diff --git a/Builder/scripts/docker-builder-backend.sh b/Builder/scripts/docker-builder-backend.sh index a94e92f..6507fa3 100755 --- a/Builder/scripts/docker-builder-backend.sh +++ b/Builder/scripts/docker-builder-backend.sh @@ -1,9 +1,5 @@ #!/bin/bash -# This script is run as root and so builder user environment variables aren't passed through -source /home/builder/registry-credentials -rm /home/builder/registry-credentials - # Test for any arguments, such as --debug for i in "$@" do diff --git a/Builder/scripts/singularity-builder-backend.sh b/Builder/scripts/singularity-builder-backend.sh index e61e388..304f22d 100755 --- a/Builder/scripts/singularity-builder-backend.sh +++ b/Builder/scripts/singularity-builder-backend.sh @@ -1,9 +1,5 @@ #!/bin/bash -# This script is run as root and so builder user environment variables aren't passed through -source /home/builder/registry-credentials -rm /home/builder/registry-credentials - # Test for any arguments, such as --debug for i in "$@" do diff --git a/BuilderQueue/include/BuilderQueue.h b/BuilderQueue/include/BuilderQueue.h index 82be1a7..24d59af 100644 --- a/BuilderQueue/include/BuilderQueue.h +++ b/BuilderQueue/include/BuilderQueue.h @@ -62,6 +62,7 @@ class BuilderQueue { // Return a json formatted string representing the status of the queue std::string status_json(); + private: asio::io_context &io_context; diff --git a/BuilderQueue/include/Connection.h b/BuilderQueue/include/Connection.h index 023ad0d..d45d0e4 100644 --- a/BuilderQueue/include/Connection.h +++ b/BuilderQueue/include/Connection.h @@ -46,6 +46,8 @@ class Connection : public std::enable_shared_from_this { void request_queue_stats(); + void request_image_list(); + void builder_ready(BuilderData builder); void wait_for_close(); diff --git a/BuilderQueue/include/Docker.h b/BuilderQueue/include/Docker.h new file mode 100644 index 0000000..2e3eb79 --- /dev/null +++ b/BuilderQueue/include/Docker.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "Logger.h" + +namespace asio = boost::asio; +namespace bp = boost::process; + +class Docker : public std::enable_shared_from_this { +public: + explicit Docker(asio::io_context &io_context) : io_context(io_context), output_pipe(io_context) {} + + // Request to get a list of docker builders + // The handler must have the form void(std::error_code error, std::string image_list) + template + void request_image_list(ListHandler handler) { + std::string list_command("/usr/local/bin/list-images.sh"); + + Logger::info("Launching command: " + list_command); + + std::error_code list_error; + process = bp::child(list_command, bp::std_in.close(), (bp::std_out & bp::std_err) > output_pipe, group, + list_error); + if (list_error) { + auto error = std::error_code(list_error.value(), std::generic_category()); + Logger::error("Error requesting image list: " + list_error.message()); + io_context.post(std::bind(handler, error, std::string("image list error"))); + } + + // Read the list-images command output until we reach EOF, which is returned as an error + auto self(shared_from_this()); + asio::async_read(output_pipe, output_buffer, + [this, self, handler](boost::system::error_code error, std::size_t) { + if (error != asio::error::eof) { + auto read_error = std::error_code(error.value(), std::generic_category()); + Logger::error("Error reading image list output: " + read_error.message()); + std::string no_list("Error reading image list output"); + io_context.post(std::bind(handler, read_error, no_list)); + } else { + // Complete the handler + std::string image_list((std::istreambuf_iterator(&output_buffer)), std::istreambuf_iterator()); + io_context.post(std::bind(handler, std::error_code(), image_list)); + Logger::info("image list posted"); + } + }); + } + +private: + asio::io_context &io_context; + bp::child process; + bp::group group; + bp::async_pipe output_pipe; + asio::streambuf output_buffer; +}; \ No newline at end of file diff --git a/BuilderQueue/scripts/list-images.sh b/BuilderQueue/scripts/list-images.sh new file mode 100755 index 0000000..c705e5e --- /dev/null +++ b/BuilderQueue/scripts/list-images.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +/usr/local/bin/docker-ls tags --progress-indicator=false --user ${DOCKERHUB_READONLY_USERNAME} --password ${DOCKERHUB_READONLY_TOKEN} olcf/titan + +/usr/local/bin/docker-ls tags --progress-indicator=false --user ${DOCKERHUB_READONLY_USERNAME} --password ${DOCKERHUB_READONLY_TOKEN} olcf/summitdev + +/usr/local/bin/docker-ls tags --progress-indicator=false --user ${DOCKERHUB_READONLY_USERNAME} --password ${DOCKERHUB_READONLY_TOKEN} olcf/summit \ No newline at end of file diff --git a/BuilderQueue/src/Connection.cpp b/BuilderQueue/src/Connection.cpp index b491d76..219d5e0 100644 --- a/BuilderQueue/src/Connection.cpp +++ b/BuilderQueue/src/Connection.cpp @@ -1,6 +1,7 @@ #include "Connection.h" #include #include +#include "Docker.h" using namespace std::placeholders; @@ -71,6 +72,23 @@ void Connection::request_queue_stats() { }); } +void Connection::request_image_list() { + // Persist this connection + auto self(shared_from_this()); + + // Get list of docker images + Logger::info("Request for image list"); + std::make_shared(stream.get_executor().context())->request_image_list([this, self](std::error_code ec, std::string image_list){ + Logger::info("Writing image list"); + stream.async_write(asio::buffer(image_list), [this, self, image_list](beast::error_code error, std::size_t bytes) { + boost::ignore_unused(bytes); + if (error) { + Logger::error("Wrote image list"); + } + }); + }); +} + void Connection::read_request_string() { // Persist this connection auto self(shared_from_this()); @@ -85,6 +103,8 @@ void Connection::read_request_string() { request_builder(); } else if(request == "queue_status_request") { request_queue_stats(); + } else if(request == "image_list_request") { + request_image_list(); } else { Logger::error("Bad initial request string: " + request); } diff --git a/CMakeLists.txt b/CMakeLists.txt index c6842c6..9bc9b68 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,8 @@ set(SOURCE_FILES_QUEUE BuilderQueue/include/OpenStack.h Common/src/BuilderData.cpp Common/include/ClientData.h - BuilderQueue/include/Server.h) + BuilderQueue/include/Server.h + BuilderQueue/include/Docker.h) set(SOURCE_FILES_BUILDER Builder/src/main.cpp @@ -47,19 +48,28 @@ set(SOURCE_FILES_STATUS Common/src/BuilderData.cpp Common/include/ClientData.h) +set(SOURCE_FILES_IMAGES + Client/src/images.cpp + Client/include/WaitingAnimation.h + Common/src/Logger.cpp + Common/include/Logger.h + Common/include/ClientData.h) + # Scripts required for runtime set(BUILDER_SCRIPTS Builder/scripts/singularity-builder-backend.sh Builder/scripts/docker-builder-backend.sh) set(QUEUE_SCRIPTS BuilderQueue/scripts/create-builder.sh - BuilderQueue/scripts/destroy-builder.sh) + BuilderQueue/scripts/destroy-builder.sh + BuilderQueue/scripts/list-images.sh) # Create executables add_executable(builder-queue ${SOURCE_FILES_QUEUE}) add_executable(builder-server ${SOURCE_FILES_BUILDER}) add_executable(container-builder ${SOURCE_FILES_CLIENT}) add_executable(cb-status ${SOURCE_FILES_STATUS}) +add_executable(cb-images ${SOURCE_FILES_IMAGES}) # Ignore system boost and use module system boost set(Boost_NO_BOOST_CMAKE TRUE) @@ -80,6 +90,8 @@ set_target_properties(container-builder PROPERTIES COMPILE_FLAGS "${COMPILE_FLAG set_target_properties(container-builder PROPERTIES LINK_FLAGS "${LINK_FLAGS} ${HARDENING_FLAGS}") set_target_properties(cb-status PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS} ${HARDENING_FLAGS}") set_target_properties(cb-status PROPERTIES LINK_FLAGS "${LINK_FLAGS} ${HARDENING_FLAGS}") +set_target_properties(cb-images PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS} ${HARDENING_FLAGS}") +set_target_properties(cb-images PROPERTIES LINK_FLAGS "${LINK_FLAGS} ${HARDENING_FLAGS}") set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) @@ -87,6 +99,7 @@ target_link_libraries(builder-queue ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(builder-server ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(container-builder ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(cb-status ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(cb-images ${CMAKE_THREAD_LIBS_INIT}) find_package(Boost 1.66.0 COMPONENTS system filesystem serialization regex thread program_options REQUIRED) @@ -95,12 +108,15 @@ target_link_libraries(builder-queue ${Boost_LIBRARIES}) target_link_libraries(builder-server ${Boost_LIBRARIES}) target_link_libraries(container-builder ${Boost_LIBRARIES}) target_link_libraries(cb-status ${Boost_LIBRARIES}) +target_link_libraries(cb-images ${Boost_LIBRARIES}) + # Install executables install(TARGETS builder-queue COMPONENT builder-queue DESTINATION bin OPTIONAL) install(TARGETS builder-server COMPONENT builder-server DESTINATION bin OPTIONAL) install(TARGETS container-builder COMPONENT client DESTINATION bin OPTIONAL) install(TARGETS cb-status COMPONENT client DESTINATION bin OPTIONAL) +install(TARGETS cb-images COMPONENT client DESTINATION bin OPTIONAL) # Install scripts install(FILES ${BUILDER_SCRIPTS} diff --git a/Client/src/images.cpp b/Client/src/images.cpp new file mode 100644 index 0000000..39adbab --- /dev/null +++ b/Client/src/images.cpp @@ -0,0 +1,89 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ClientData.h" +#include "WaitingAnimation.h" +#include "pwd.h" + +namespace asio = boost::asio; +using asio::ip::tcp; +namespace beast = boost::beast; +namespace websocket = beast::websocket; + +void parse_environment(ClientData &client_data) { + Logger::debug("Parsing environment variables"); + + struct passwd *pws; + pws = getpwuid(getuid()); + if (pws == NULL) { + throw std::runtime_error("Error get login name for client data!"); + } + client_data.user_id = std::string(pws->pw_name); + + auto host = std::getenv("QUEUE_HOST"); + if (!host) { + throw std::runtime_error("QUEUE_HOST not set!"); + } + client_data.queue_host = std::string(host); +} + +void print_images(websocket::stream &queue_stream) { + Logger::debug("Writing image list request string"); + std::string request_string("image_list_request"); + queue_stream.write(asio::buffer(request_string)); + + Logger::debug("Read image list string"); + std::string images_string; + auto images_buffer = boost::asio::dynamic_buffer(images_string); + queue_stream.read(images_buffer); + + std::cout << images_string; +} + +int main(int argc, char *argv[]) { + asio::io_context io_context; + websocket::stream queue_stream(io_context); + + try { + ClientData client_data; + + parse_environment(client_data); + + WaitingAnimation wait_queue("Connecting to BuilderQueue"); + // Open a WebSocket stream to the queue + tcp::resolver queue_resolver(io_context); + asio::connect(queue_stream.next_layer(), queue_resolver.resolve({client_data.queue_host, "8080"})); + queue_stream.handshake(client_data.queue_host + ":8080", "/"); + wait_queue.stop_success("Connected to queue: " + client_data.queue_host); + + // Request a build host from the queue + WaitingAnimation wait_builder("Requesting image list"); + print_images(queue_stream); + + } catch (const boost::exception &ex) { + auto diagnostics = diagnostic_information(ex); + Logger::error(std::string() + "Container Builder exception encountered: " + diagnostics); + } catch (const std::exception &ex) { + Logger::error(std::string() + "Container Builder exception encountered: " + ex.what()); + } catch (...) { + Logger::error("Unknown exception caught!"); + } + + // Attempt to disconnect from builder and queue + try { + Logger::debug("Attempting normal close"); + queue_stream.close(websocket::close_code::normal); + } catch (...) { + Logger::debug("Failed to cleanly close the WebSockets"); + } + + return 0; +} \ No newline at end of file diff --git a/Deployment/create-builder-image.sh b/Deployment/create-builder-image.sh index 5265bd9..5f18faa 100755 --- a/Deployment/create-builder-image.sh +++ b/Deployment/create-builder-image.sh @@ -66,13 +66,14 @@ echo "Provisioning the builder" ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/provision-builder.sh # Copy readonly credentials to the builder, these variables must be set in the gitlab runner that's running this script -echo "GITLAB_READONLY_USERNAME=${GITLAB_READONLY_USERNAME}" > ./registry-credentials -echo "GITLAB_READONLY_TOKEN=${GITLAB_READONLY_TOKEN}" >> ./registry-credentials -echo "DOCKERHUB_READONLY_USERNAME=${DOCKERHUB_READONLY_USERNAME}" >> ./registry-credentials -echo "DOCKERHUB_READONLY_TOKEN=${DOCKERHUB_READONLY_TOKEN}" >> ./registry-credentials -scp -o StrictHostKeyChecking=no -i ${KEY_FILE} ./registry-credentials cades@${VM_IP}:/home/cades/registry-credentials -ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo mv /home/cades/registry-credentials /home/builder/registry-credentials' -ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo chown builder /home/builder/registry-credentials' +echo "GITLAB_READONLY_USERNAME=${GITLAB_READONLY_USERNAME}" > ./environment.sh +echo "GITLAB_READONLY_TOKEN=${GITLAB_READONLY_TOKEN}" >> ./environment.sh +echo "DOCKERHUB_READONLY_USERNAME=${DOCKERHUB_READONLY_USERNAME}" >> ./environment.sh +echo "DOCKERHUB_READONLY_TOKEN=${DOCKERHUB_READONLY_TOKEN}" >> ./environment.sh +echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib" >> ./environment +scp -o StrictHostKeyChecking=no -i ${KEY_FILE} ./environment.sh cades@${VM_IP}:/home/cades/environment.sh +ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo mv /home/cades/environment.sh /home/builder/environment.sh' +ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo chown builder /home/builder/environment.sh' echo "Reboot the server to ensure its in a clean state before creating the snapshot" openstack server reboot --wait ${VM_UUID} diff --git a/Deployment/create-queue.sh b/Deployment/create-queue.sh index 055e346..8d876a2 100755 --- a/Deployment/create-queue.sh +++ b/Deployment/create-queue.sh @@ -45,12 +45,15 @@ ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < $ echo "Provisioning the queue" ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo bash -s' < ${SCRIPT_DIR}/provision-queue.sh -# Copy OpenStack credentials to VM and then move to correct directory -# These credentials are available as environment variables to the runners -printenv | grep ^OS_ > ./openrc.sh # "Reconstruct" openrc.sh -scp -o StrictHostKeyChecking=no -i ${KEY_FILE} ./openrc.sh cades@${VM_IP}:/home/cades/openrc.sh -ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo mv /home/cades/openrc.sh /home/queue/openrc.sh' -ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo chown queue /home/queue/openrc.sh' +# Copy OpenStack and Docker registry credentials to VM and then move to correct directory +# These credentials are available as environment variables to the gitlab runners +printenv | grep ^OS_ > ./environment.sh # "Reconstruct" openrc.sh +echo "DOCKERHUB_READONLY_USERNAME=${DOCKERHUB_READONLY_USERNAME}" >> ./environment.sh +echo "DOCKERHUB_READONLY_TOKEN=${DOCKERHUB_READONLY_TOKEN}" >> ./environment.sh +echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib" >> ./environment.sh +scp -o StrictHostKeyChecking=no -i ${KEY_FILE} ./openrc.sh cades@${VM_IP}:/home/cades/environment.sh +ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo mv /home/cades/environment.sh /home/queue/environment.sh' +ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} cades@${VM_IP} 'sudo chown queue /home/queue/environment.sh' # Reboot to start queue service added in provisioning openstack server reboot --wait ${VM_UUID} diff --git a/Deployment/deploy-summit.sh b/Deployment/deploy-summit.sh index f0d356c..1375fdc 100755 --- a/Deployment/deploy-summit.sh +++ b/Deployment/deploy-summit.sh @@ -38,6 +38,7 @@ rm -rf build && mkdir build && cd build CC=gcc CXX=g++ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${SW_ROOT} .. make container-builder make cb-status +make images cmake -DCOMPONENT=client -P cmake_install.cmake # Generate a public modulefile diff --git a/Deployment/deploy-summitdev.sh b/Deployment/deploy-summitdev.sh index 48b819b..05ceee1 100755 --- a/Deployment/deploy-summitdev.sh +++ b/Deployment/deploy-summitdev.sh @@ -38,6 +38,7 @@ rm -rf build && mkdir build && cd build CC=gcc CXX=g++ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${SW_ROOT} .. make container-builder make cb-status +make images cmake -DCOMPONENT=client -P cmake_install.cmake # Generate a public modulefile diff --git a/Deployment/deploy-titan.sh b/Deployment/deploy-titan.sh index 10f4307..aad43c1 100755 --- a/Deployment/deploy-titan.sh +++ b/Deployment/deploy-titan.sh @@ -39,6 +39,7 @@ rm -rf build && mkdir build && cd build CC=gcc CXX=g++ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${SW_ROOT} .. make container-builder make cb-status +make images cmake -DCOMPONENT=client -P cmake_install.cmake # Generate a public modulefile diff --git a/Deployment/provision-builder.sh b/Deployment/provision-builder.sh index 3e1d14a..b480493 100755 --- a/Deployment/provision-builder.sh +++ b/Deployment/provision-builder.sh @@ -112,7 +112,7 @@ After=network.target Type=simple User=builder WorkingDirectory=/home/builder -Environment="LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib" +EnvironmentFile=/home/queue/environment.sh ExecStart=/usr/local/bin/builder-server Restart=no diff --git a/Deployment/provision-queue.sh b/Deployment/provision-queue.sh index 722c3bd..97bc611 100755 --- a/Deployment/provision-queue.sh +++ b/Deployment/provision-queue.sh @@ -47,6 +47,12 @@ rm -rf /container-builder # Install OpenStack command line client pip install python-openstackclient +# Install docker-ls +wget https://github.com/mayflower/docker-ls/releases/download/v0.3.1/docker-ls-linux-amd64.zip +unzip docker-ls-linux-amd64 +mv docker-ls /usr/local/bin +rm docker-rm + # Create systemd script and launch the BuilderQueue daemon cat << EOF > /etc/systemd/system/builder-queue.service [Unit] @@ -56,8 +62,7 @@ After=network.target [Service] Type=simple User=queue -EnvironmentFile=/home/queue/openrc.sh -Environment="LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib" +EnvironmentFile=/home/queue/environment.sh WorkingDirectory=/home/queue ExecStart=/usr/local/bin/builder-queue Restart=no