From 70f62324da8a1df9bebdcfc6c14413cfcadf0a71 Mon Sep 17 00:00:00 2001 From: atj Date: Wed, 17 Jan 2018 12:28:47 -0500 Subject: [PATCH] Add ability to add debug output to docker/singularity commands at builder level. Fix, hopefully, gitlab docker registry building --- Builder/src/main.cpp | 46 ++++++++++++---------- BuilderQueue/include/BuilderQueue.h | 14 ++++--- BuilderQueue/include/Connection.h | 7 ++-- BuilderQueue/include/OpenStack.h | 4 +- BuilderQueue/include/Server.h | 13 ++++--- BuilderQueue/src/BuilderQueue.cpp | 32 ++++++++-------- BuilderQueue/src/Connection.cpp | 21 +++++----- BuilderQueue/src/main.cpp | 4 +- Client/include/WaitingAnimation.h | 18 ++++----- Client/src/main.cpp | 51 +++++++++++++------------ Common/include/BuilderData.h | 4 +- Common/include/ClientData.h | 7 +++- Common/include/Logger.h | 59 ++++++++++++++++------------- Common/src/BuilderData.cpp | 4 +- Common/src/Logger.cpp | 4 +- Scripts/DockerBuilderBackend | 32 ++++++++++++---- Scripts/SingularityBuilderBackend | 27 +++++++++++-- 17 files changed, 205 insertions(+), 142 deletions(-) diff --git a/Builder/src/main.cpp b/Builder/src/main.cpp index e8a0a14..ab80f12 100644 --- a/Builder/src/main.cpp +++ b/Builder/src/main.cpp @@ -19,9 +19,9 @@ namespace beast = boost::beast; namespace websocket = beast::websocket; namespace bp = boost::process; -using callback_type = std::function; +using callback_type = std::function; -ClientData read_client_data(websocket::stream& client_stream) { +ClientData read_client_data(websocket::stream &client_stream) { Logger::info("Reading client data"); Logger::info("Reading serialized client data"); @@ -39,11 +39,11 @@ ClientData read_client_data(websocket::stream& client_stream) { return client_data; } -void read_file(websocket::stream& client_stream, - const std::string& file_name) { +void read_file(websocket::stream &client_stream, + const std::string &file_name) { Logger::info("Opening " + file_name + " for writing"); std::ofstream file; - file.exceptions ( std::ofstream::failbit | std::ofstream::badbit ); + file.exceptions(std::ofstream::failbit | std::ofstream::badbit); file.open(file_name, std::fstream::out | std::fstream::binary | std::fstream::trunc); // Read in file from websocket in chunks @@ -74,14 +74,22 @@ void read_file(websocket::stream& client_stream, Logger::info(file_name + " successfully read file"); } -std::string build_command(const ClientData& client_data) { +std::string build_command(const ClientData &client_data) { std::string build_command; + // Basic build command if (client_data.backend == BackendType::singularity) { build_command = "/usr/bin/sudo /usr/local/bin/SingularityBuilderBackend"; - } else if(client_data.backend == BackendType::docker) { + } else if (client_data.backend == BackendType::docker) { build_command = "/usr/bin/sudo /usr/local/bin/DockerBuilderBackend"; + } else { + throw std::runtime_error("Invalid builder backend"); + } + + // Enable debug output if requested + if (client_data.log_priority >= LogPriority::debug) { + build_command += " --debug"; } Logger::info("Build command prepared: " + build_command); @@ -89,11 +97,11 @@ std::string build_command(const ClientData& client_data) { return build_command; } -void stream_build(websocket::stream& client_stream, - const std::string& build_command) { +void stream_build(websocket::stream &client_stream, + const std::string &build_command) { Logger::info("Starting subprocess and redirecting output to pipe"); - auto& io_context = client_stream.get_executor().context(); + auto &io_context = client_stream.get_executor().context(); bp::async_pipe std_pipe{io_context}; bp::group group; bp::child build_child(build_command, bp::std_in.close(), (bp::std_out & bp::std_err) > std_pipe, group); @@ -103,9 +111,9 @@ void stream_build(websocket::stream& client_stream, const auto max_read_bytes = 4096; std::array std_buffer; - callback_type stream_output = [&] (const boost::system::error_code& error, - std::size_t bytes_read) { - if(error == asio::error::eof) { + callback_type stream_output = [&](const boost::system::error_code &error, + std::size_t bytes_read) { + if (error == asio::error::eof) { client_stream.write_some(true, asio::buffer(std_buffer.data(), bytes_read)); return; } else { @@ -125,19 +133,19 @@ void stream_build(websocket::stream& client_stream, auto exit_code_string = std::to_string(exit_code); client_stream.write(asio::buffer(exit_code_string)); - if(exit_code != 0) { + if (exit_code != 0) { throw std::runtime_error("Container build failed"); } Logger::info("Done streaming subprocess output"); } -void write_file(websocket::stream& client_stream, - const std::string& file_name) { +void write_file(websocket::stream &client_stream, + const std::string &file_name) { Logger::info("Opening " + file_name + " for reading"); std::ifstream container; - container.exceptions ( std::ofstream::failbit | std::ofstream::badbit ); + container.exceptions(std::ofstream::failbit | std::ofstream::badbit); container.open(file_name, std::fstream::in | std::fstream::binary); const auto file_size = boost::filesystem::file_size(file_name); @@ -153,7 +161,7 @@ void write_file(websocket::stream& client_stream, container.read(chunk_buffer.data(), chunk_size); container_csc.process_bytes(chunk_buffer.data(), chunk_size); bytes_remaining -= chunk_size; - if(bytes_remaining == 0) + if (bytes_remaining == 0) fin = true; client_stream.write_some(fin, asio::buffer(chunk_buffer.data(), chunk_size)); } while (!fin); @@ -210,7 +218,7 @@ int main(int argc, char *argv[]) { // Attempt to close connection try { client_stream.close(websocket::close_code::normal); - } catch(...) { + } catch (...) { Logger::error("Failed to cleanly close the WebSocket"); } return 0; diff --git a/BuilderQueue/include/BuilderQueue.h b/BuilderQueue/include/BuilderQueue.h index 927a6cb..5e7a334 100644 --- a/BuilderQueue/include/BuilderQueue.h +++ b/BuilderQueue/include/BuilderQueue.h @@ -9,7 +9,7 @@ namespace asio = boost::asio; -using FetchHandler = std::function; +using FetchHandler = std::function; // Note that the handler, from Connection, contains 'self' which maintains the lifetime of the connection instance // If a client disconnects while the handler is pending it won't be cleaned up @@ -24,10 +24,13 @@ class BuilderQueue { } // Don't allow the queue to be copied or moved - BuilderQueue(const BuilderQueue&) =delete; - BuilderQueue& operator=(const BuilderQueue&) =delete; - BuilderQueue(BuilderQueue&&) noexcept =delete; - BuilderQueue& operator=(BuilderQueue&&) =delete; + BuilderQueue(const BuilderQueue &) = delete; + + BuilderQueue &operator=(const BuilderQueue &) = delete; + + BuilderQueue(BuilderQueue &&) noexcept = delete; + + BuilderQueue &operator=(BuilderQueue &&) = delete; // Add the specified handler to the queue @@ -46,6 +49,7 @@ class BuilderQueue { // Create reserve builders as needed void create_reserve_builders(); + private: asio::io_context &io_context; diff --git a/BuilderQueue/include/Connection.h b/BuilderQueue/include/Connection.h index 45b256a..d055dc7 100644 --- a/BuilderQueue/include/Connection.h +++ b/BuilderQueue/include/Connection.h @@ -13,12 +13,11 @@ namespace websocket = beast::websocket; class Connection : public std::enable_shared_from_this { public: - explicit Connection(tcp::socket socket, BuilderQueue& queue) : stream(std::move(socket)), - queue(queue) - {}; + explicit Connection(tcp::socket socket, BuilderQueue &queue) : stream(std::move(socket)), + queue(queue) {}; ~Connection() { - if(builder) { + if (builder) { queue.return_builder(builder.get()); } Logger::info("Connection ending"); diff --git a/BuilderQueue/include/OpenStack.h b/BuilderQueue/include/OpenStack.h index b8e41ba..2c00ae7 100644 --- a/BuilderQueue/include/OpenStack.h +++ b/BuilderQueue/include/OpenStack.h @@ -18,8 +18,8 @@ namespace pt = boost::property_tree; class OpenStack : public std::enable_shared_from_this { public: - OpenStack(asio::io_context &io_context) : io_context(io_context), - output_pipe(io_context) {} + explicit OpenStack(asio::io_context &io_context) : io_context(io_context), + output_pipe(io_context) {} // Request to create a new builder // The handler must have the form void(std::error_code error, BuilderData builder) diff --git a/BuilderQueue/include/Server.h b/BuilderQueue/include/Server.h index 0621977..3dea33a 100644 --- a/BuilderQueue/include/Server.h +++ b/BuilderQueue/include/Server.h @@ -10,17 +10,20 @@ using asio::ip::tcp; class Server { public: - Server(asio::io_context &io_context, BuilderQueue& queue) : + Server(asio::io_context &io_context, BuilderQueue &queue) : acceptor(io_context, tcp::endpoint(tcp::v4(), 8080)), queue(queue) { accept_connection(); }; // Don't allow the connection server to be copied or moved - Server(const Server&) =delete; - Server& operator=(const Server&) =delete; - Server(Server&&) noexcept =delete; - Server& operator=(Server&&) =delete; + Server(const Server &) = delete; + + Server &operator=(const Server &) = delete; + + Server(Server &&) noexcept = delete; + + Server &operator=(Server &&) = delete; private: tcp::acceptor acceptor; diff --git a/BuilderQueue/src/BuilderQueue.cpp b/BuilderQueue/src/BuilderQueue.cpp index 07325e9..2c5c4dc 100644 --- a/BuilderQueue/src/BuilderQueue.cpp +++ b/BuilderQueue/src/BuilderQueue.cpp @@ -20,7 +20,7 @@ void BuilderQueue::return_builder(BuilderData builder) { void BuilderQueue::process_pending_handler() { Logger::info("Processing " + std::to_string(pending_handlers.size()) + - " handlers with " + std::to_string(reserve_builders.size()) + " reserve builders"); + " handlers with " + std::to_string(reserve_builders.size()) + " reserve builders"); if (!pending_handlers.empty() && !reserve_builders.empty()) { Logger::info("Processing pending handler"); @@ -39,7 +39,7 @@ void BuilderQueue::process_pending_handler() { Logger::info("Providing builder to client: " + builder.id); // Post the handler to pass the builder to the connection - asio::post(io_context, std::bind(handler,builder)); + asio::post(io_context, std::bind(handler, builder)); // Attempt to create a new reserve builder if required create_reserve_builders(); @@ -47,7 +47,8 @@ void BuilderQueue::process_pending_handler() { } void BuilderQueue::create_reserve_builders() { - Logger::info("Checking reserve builder count with " + std::to_string(reserve_builders.size()) + " reserve builders"); + Logger::info( + "Checking reserve builder count with " + std::to_string(reserve_builders.size()) + " reserve builders"); // Attempt to create the required number of reserve builders while staying below the total allowed builder count auto potential_reserve_count = reserve_builders.size() + outstanding_create_count; @@ -68,18 +69,19 @@ void BuilderQueue::create_reserve_builders() { for (std::size_t i = 0; i < request_count; i++) { Logger::info("Attempting to create builder " + std::to_string(i)); - std::make_shared(io_context)->request_create([this, i](std::error_code error, BuilderData builder) { - if (!error) { - outstanding_create_count--; - Logger::info("Created builder " + std::to_string(i) + ": " + builder.id); - reserve_builders.push_back(builder); - process_pending_handler(); - } else { - Logger::error("Error creating builder, retrying in five seconds: " + std::to_string(i)); - asio::deadline_timer timer(io_context, boost::posix_time::seconds(5)); - timer.async_wait(std::bind(&BuilderQueue::create_reserve_builders, this)); - } - }); + std::make_shared(io_context)->request_create( + [this, i](std::error_code error, BuilderData builder) { + if (!error) { + outstanding_create_count--; + Logger::info("Created builder " + std::to_string(i) + ": " + builder.id); + reserve_builders.push_back(builder); + process_pending_handler(); + } else { + Logger::error("Error creating builder, retrying in five seconds: " + std::to_string(i)); + asio::deadline_timer timer(io_context, boost::posix_time::seconds(5)); + timer.async_wait(std::bind(&BuilderQueue::create_reserve_builders, this)); + } + }); } } } \ No newline at end of file diff --git a/BuilderQueue/src/Connection.cpp b/BuilderQueue/src/Connection.cpp index 2a0e638..bed4e83 100644 --- a/BuilderQueue/src/Connection.cpp +++ b/BuilderQueue/src/Connection.cpp @@ -1,7 +1,6 @@ #include "Connection.h" #include #include -#include using namespace std::placeholders; @@ -10,10 +9,10 @@ void Connection::wait_for_close() { auto self(shared_from_this()); Logger::info("Waiting for connection to close"); - stream.async_read(buffer, [this, self] (beast::error_code error, std::size_t bytes) { + stream.async_read(buffer, [this, self](beast::error_code error, std::size_t bytes) { boost::ignore_unused(bytes); - if(error == websocket::error::closed) { + if (error == websocket::error::closed) { Logger::info("Cleaning closing connection"); } else { Logger::error("Error closing connection: " + error.message()); @@ -34,9 +33,9 @@ void Connection::builder_ready(BuilderData builder) { std::string request_string(serialized_builder); Logger::info("Writing builder: " + builder.id); - stream.async_write(asio::buffer(request_string), [this, self, builder] (beast::error_code error, std::size_t bytes){ + stream.async_write(asio::buffer(request_string), [this, self, builder](beast::error_code error, std::size_t bytes) { boost::ignore_unused(bytes); - if(!error) { + if (!error) { wait_for_close(); } else { Logger::error("Error writing builder: " + builder.id); @@ -51,7 +50,7 @@ void Connection::request_builder() { Logger::info("Request to checkout builder made"); // Pass the provide_builder callback to the queue // 'self' is passed to keep the connection alive as long as it's waiting for a builder in the queue - queue.checkout_builder([this, self] (BuilderData builder) { + queue.checkout_builder([this, self](BuilderData builder) { builder_ready(builder); }); } @@ -61,12 +60,12 @@ void Connection::read_request_string() { auto self(shared_from_this()); Logger::info("Reading initial request"); - stream.async_read(buffer, [this, self] (beast::error_code error, std::size_t bytes) { + stream.async_read(buffer, [this, self](beast::error_code error, std::size_t bytes) { boost::ignore_unused(bytes); - if(!error) { + if (!error) { auto request = beast::buffers_to_string(buffer.data()); buffer.consume(buffer.size()); - if(request == "checkout_builder_request") { + if (request == "checkout_builder_request") { request_builder(); } else { Logger::error("Bad initial request string: " + request); @@ -83,8 +82,8 @@ void Connection::start() { // Accept websocket handshake Logger::info("waiting for client WebSocket handshake"); - stream.async_accept([this, self](beast::error_code error){ - if(!error) { + stream.async_accept([this, self](beast::error_code error) { + if (!error) { Logger::info("Setting stream to binary mode"); stream.binary(true); read_request_string(); diff --git a/BuilderQueue/src/main.cpp b/BuilderQueue/src/main.cpp index 1b161e5..a433972 100644 --- a/BuilderQueue/src/main.cpp +++ b/BuilderQueue/src/main.cpp @@ -17,9 +17,9 @@ int main(int argc, char *argv[]) { } catch (const boost::exception &ex) { auto diagnostics = diagnostic_information(ex); Logger::error(std::string() + "io_service exception encountered: " + diagnostics); - } catch (const std::exception& ex) { + } catch (const std::exception &ex) { Logger::error(std::string() + "io_service exception encountered: " + ex.what()); - } catch(...) { + } catch (...) { Logger::error("Unknown io_service exception encountered"); } } diff --git a/Client/include/WaitingAnimation.h b/Client/include/WaitingAnimation.h index ffe639d..56372e6 100644 --- a/Client/include/WaitingAnimation.h +++ b/Client/include/WaitingAnimation.h @@ -11,14 +11,14 @@ using namespace std::chrono_literals; // Print animated ellipses, used to indicate to the user we're waiting on an async routine class WaitingAnimation { public: - WaitingAnimation(const std::string &message) : active(true) { + explicit WaitingAnimation(const std::string &message) : active(true) { const auto prefix = message + ": "; // If an error occurs before the thread is created still print a message std::cout << prefix << ". "; // If debug logging is enabled don't do any animation as it might interfere with debug printing - if(Logger::get_max_priority() >= LogPriority::info) { + if (Logger::get_priority() >= LogPriority::info) { return; } @@ -44,7 +44,7 @@ class WaitingAnimation { } ~WaitingAnimation() { - if(active) { + if (active) { stop_error("Error encountered"); } } @@ -53,26 +53,26 @@ class WaitingAnimation { void stop_success(const std::string &message) { // Stop animation and join thread active = false; - if(animation.joinable()) + if (animation.joinable()) animation.join(); // Move cursor to start of initial animation message - std::cout<<"\b\b\b"; + std::cout << "\b\b\b"; - std::cout<& builder_stream, ClientData client_data) { +void write_client_data(websocket::stream &builder_stream, ClientData client_data) { Logger::debug("Writing client data"); Logger::debug("Serializing client data string"); @@ -34,10 +33,10 @@ void write_client_data(websocket::stream& builder_stream, ClientDat builder_stream.write(asio::buffer(serialized_client_data)); } -void read_file(websocket::stream& stream, const std::string& file_name) { +void read_file(websocket::stream &stream, const std::string &file_name) { Logger::debug("Opening " + file_name + " for writing"); std::ofstream file; - file.exceptions ( std::ofstream::failbit | std::ofstream::badbit ); + file.exceptions(std::ofstream::failbit | std::ofstream::badbit); file.open(file_name, std::fstream::out | std::fstream::binary | std::fstream::trunc); // Read in file from websocket in chunks @@ -68,12 +67,12 @@ void read_file(websocket::stream& stream, const std::string& file_n Logger::debug(file_name + " successfully read"); } -void write_file(websocket::stream& stream, - const std::string& file_name) { +void write_file(websocket::stream &stream, + const std::string &file_name) { Logger::debug("Opening " + file_name + " for reading"); std::ifstream container; - container.exceptions ( std::ofstream::failbit | std::ofstream::badbit ); + container.exceptions(std::ofstream::failbit | std::ofstream::badbit); container.open(file_name, std::fstream::in | std::fstream::binary); const auto file_size = boost::filesystem::file_size(file_name); @@ -89,7 +88,7 @@ void write_file(websocket::stream& stream, container.read(chunk_buffer.data(), chunk_size); container_csc.process_bytes(chunk_buffer.data(), chunk_size); bytes_remaining -= chunk_size; - if(bytes_remaining == 0) + if (bytes_remaining == 0) fin = true; stream.write_some(fin, asio::buffer(chunk_buffer.data(), chunk_size)); } while (!fin); @@ -113,11 +112,11 @@ void parse_arguments(ClientData &client_data, int argc, char **argv) { ("help", "produce help message") ("debug", po::bool_switch(), "enable debug information") ("arch", po::value()->default_value("x86_64"), - "select architecture, valid options are x86_64 and ppc64le") + "select architecture, valid options are x86_64 and ppc64le") ("backend", po::value()->default_value("singularity"), - "select the builder backend to use, valid options are singularity and docker") + "select the builder backend to use, valid options are singularity and docker") ("tty", po::value()->default_value(isatty(fileno(stdout))), - "true if the data should be presented as if a tty is present") + "true if the data should be presented as if a tty is present") ("container", po::value()->required(), "(required) the container name") ("definition", po::value()->required(), "(required) the definition file"); @@ -136,18 +135,19 @@ void parse_arguments(ClientData &client_data, int argc, char **argv) { exit(EXIT_FAILURE); } - // Set class variables based upon program arguments + // Enable debug information + if (vm["debug"].as()) { + Logger::set_priority(LogPriority::debug); + Logger::debug("Debug logging enabled"); + } + + // Set client data based upon program arguments client_data.definition_path = vm["definition"].as(); client_data.container_path = vm["container"].as(); client_data.tty = vm["tty"].as(); client_data.arch = Arch::to_arch(vm["arch"].as()); client_data.backend = Backend::to_backend(vm["backend"].as()); - - // Enable debug information - if(vm["debug"].as()) { - Logger::set_max_priority(LogPriority::debug); - Logger::debug("Debug logging enabled"); - } + client_data.log_priority = Logger::get_priority(); // Make sure variables are set as required po::notify(vm); @@ -170,7 +170,7 @@ void parse_environment(ClientData &client_data) { client_data.queue_host = std::string(host); } -BuilderData get_builder(websocket::stream& queue_stream) { +BuilderData get_builder(websocket::stream &queue_stream) { Logger::debug("Writing builder request string"); std::string request_string("checkout_builder_request"); queue_stream.write(asio::buffer(request_string)); @@ -189,7 +189,7 @@ BuilderData get_builder(websocket::stream& queue_stream) { return builder_data; } -void stream_build(websocket::stream& builder_stream) { +void stream_build(websocket::stream &builder_stream) { Logger::info("Beginning to stream build"); const auto max_read_bytes = 4096; std::array buffer; @@ -205,7 +205,7 @@ void stream_build(websocket::stream& builder_stream) { auto exit_code_buffer = boost::asio::dynamic_buffer(exit_code_string); builder_stream.read(exit_code_buffer); Logger::debug("Build exit code: " + exit_code_string); - if(exit_code_string != "0") { + if (exit_code_string != "0") { throw std::runtime_error("Build failed with exit code " + exit_code_string); } } @@ -215,7 +215,7 @@ int main(int argc, char *argv[]) { std::cout.setf(std::ios::unitbuf); // Hide the the cursor - std::cout<<"\e[?25l"; + std::cout << "\e[?25l"; asio::io_context io_context; websocket::stream queue_stream(io_context); @@ -243,7 +243,8 @@ int main(int argc, char *argv[]) { builder_stream.handshake(builder_data.host + ":8080", "/"); wait_builder.stop_success("Connected to remote builder: " + builder_data.host); - Logger::debug("Setting the builder websocket stream to handle binary data and have an unlimited(uint64_t) message size"); + Logger::debug( + "Setting the builder websocket stream to handle binary data and have an unlimited(uint64_t) message size"); builder_stream.binary(true); builder_stream.read_message_max(0); @@ -272,12 +273,12 @@ int main(int argc, char *argv[]) { try { builder_stream.close(websocket::close_code::normal); queue_stream.close(websocket::close_code::normal); - } catch(...) { + } catch (...) { Logger::debug("Failed to cleanly close the WebSockets"); } // Show the cursor - std::cout<<"\e[?25h"; + std::cout << "\e[?25h"; return 0; } \ No newline at end of file diff --git a/Common/include/BuilderData.h b/Common/include/BuilderData.h index b2fbae1..254791e 100644 --- a/Common/include/BuilderData.h +++ b/Common/include/BuilderData.h @@ -14,9 +14,9 @@ class BuilderData { std::string id; }; -bool operator <(const BuilderData &lhs, const BuilderData &rhs); +bool operator<(const BuilderData &lhs, const BuilderData &rhs); -bool operator ==(const BuilderData &lhs, const BuilderData &rhs); +bool operator==(const BuilderData &lhs, const BuilderData &rhs); namespace boost { namespace serialization { diff --git a/Common/include/ClientData.h b/Common/include/ClientData.h index aeb99d3..98c9dad 100644 --- a/Common/include/ClientData.h +++ b/Common/include/ClientData.h @@ -6,6 +6,7 @@ #include #include #include +#include "Logger.h" enum class ArchType { x86_64, @@ -18,7 +19,7 @@ enum class BackendType { }; namespace Backend { - static BackendType to_backend(const std::string& backend_string) { + static BackendType to_backend(const std::string &backend_string) { if (backend_string == "singularity") return BackendType::singularity; else if (backend_string == "docker") @@ -29,7 +30,7 @@ namespace Backend { } namespace Arch { - static ArchType to_arch(const std::string& arch_string) { + static ArchType to_arch(const std::string &arch_string) { if (arch_string == "x86_64") return ArchType::x86_64; else if (arch_string == "ppc64le") @@ -43,6 +44,7 @@ class ClientData { public: std::string user_id; bool tty; + LogPriority log_priority; ArchType arch; BackendType backend; std::string container_path; @@ -58,6 +60,7 @@ namespace boost { boost::ignore_unused(version); ar & client_data.user_id; ar & client_data.tty; + ar & client_data.log_priority; ar & client_data.arch; ar & client_data.backend; ar & client_data.container_path; diff --git a/Common/include/Logger.h b/Common/include/Logger.h index ba16258..d19d762 100644 --- a/Common/include/Logger.h +++ b/Common/include/Logger.h @@ -4,69 +4,76 @@ #include enum class LogPriority { - error = 0, - success = 1, - warning = 2, - info = 3, - debug =4 + error = 0, + success = 1, + warning = 2, + info = 3, + debug = 4 }; // Logger singleton, not designed to be thread safe class Logger { public: Logger() : has_tty(isatty(STDOUT_FILENO)), - max_priority(LogPriority::info) - {} + max_priority(LogPriority::info) {} - static void set_max_priority(LogPriority priority) { + static void set_priority(LogPriority priority) { instance().max_priority = priority; } - static LogPriority get_max_priority() { + static LogPriority get_priority() { return instance().max_priority; } // Return the instance of the logger object - static Logger& instance() { + static Logger &instance() { static Logger instance; return instance; } // Print a message to the logger - void print(const std::string& message, LogPriority priority) const; + void print(const std::string &message, LogPriority priority) const; // Methods to be used to print - static void error(const std::string& message) { - const auto& logger = instance(); - if(logger.max_priority >= LogPriority::error) + static void error(const std::string &message) { + const auto &logger = instance(); + if (logger.max_priority >= LogPriority::error) logger.print(message, LogPriority::error); } - static void warning(const std::string& message) { - const auto& logger = instance(); - if(logger.max_priority >= LogPriority::warning) + + static void warning(const std::string &message) { + const auto &logger = instance(); + if (logger.max_priority >= LogPriority::warning) logger.print(message, LogPriority::warning); } - static void success(const std::string& message) { - const auto& logger = instance(); - if(logger.max_priority >= LogPriority::success) + + static void success(const std::string &message) { + const auto &logger = instance(); + if (logger.max_priority >= LogPriority::success) logger.print(message, LogPriority::success); } - static void info(const std::string& message) { - const auto& logger = instance(); - if(logger.max_priority >= LogPriority::info) + + static void info(const std::string &message) { + const auto &logger = instance(); + if (logger.max_priority >= LogPriority::info) logger.print(message, LogPriority::info); } - static void debug(const std::string& message) { - const auto& logger = instance(); - if(logger.max_priority >= LogPriority::debug) + + static void debug(const std::string &message) { + const auto &logger = instance(); + if (logger.max_priority >= LogPriority::debug) logger.print(message, LogPriority::debug); } private: void set_color(LogPriority priority) const; + void unset_color() const; + void print_prefix(LogPriority priority) const; + void print_suffix() const; + const bool has_tty; LogPriority max_priority; // The maximum priority to print }; \ No newline at end of file diff --git a/Common/src/BuilderData.cpp b/Common/src/BuilderData.cpp index c95d4ff..fffd2ed 100644 --- a/Common/src/BuilderData.cpp +++ b/Common/src/BuilderData.cpp @@ -1,9 +1,9 @@ #include "BuilderData.h" -bool operator <(const BuilderData &lhs, const BuilderData &rhs) { +bool operator<(const BuilderData &lhs, const BuilderData &rhs) { return lhs.id < rhs.id; } -bool operator ==(const BuilderData &lhs, const BuilderData &rhs) { +bool operator==(const BuilderData &lhs, const BuilderData &rhs) { return lhs.id < rhs.id; } \ No newline at end of file diff --git a/Common/src/Logger.cpp b/Common/src/Logger.cpp index 7a245a6..ecd5dd4 100644 --- a/Common/src/Logger.cpp +++ b/Common/src/Logger.cpp @@ -3,7 +3,7 @@ // Set terminal color if we're outputing to a tty void Logger::set_color(LogPriority priority) const { - if(has_tty) { + if (has_tty) { switch (priority) { case LogPriority::error: std::clog << "\033[31m"; @@ -59,7 +59,7 @@ void Logger::unset_color() const { } // Print a log message -void Logger::print(const std::string& message, LogPriority priority) const { +void Logger::print(const std::string &message, LogPriority priority) const { set_color(priority); print_prefix(priority); diff --git a/Scripts/DockerBuilderBackend b/Scripts/DockerBuilderBackend index 8722879..f292689 100755 --- a/Scripts/DockerBuilderBackend +++ b/Scripts/DockerBuilderBackend @@ -1,18 +1,36 @@ #!/bin/bash -set -e +# Test for any arguments, such as --debug +for i in "$@" +do +case ${i} in + --debug) + DEBUG_FLAG='--debug' + shift # past argument with no value + ;; + *) + echo "unknown argument to SingularityBuilderBackend" + exit 1 + ;; +esac +done -# Provide readonly access to the private gitlab docker repository -docker login code.ornl.gov:4567 -u atj -p $(cat /home/builder/container-registry-token) +# Provide readonly access to the private gitlab docker repository if using the container-recipes docker registry +grep 'code.ornl.gov:4567' ./container.def +GREP_RC=$? +if [[ $GREP_RC -eq 0 ]] ; then + echo "Using container recipes docker registry login credentials" + docker ${DEBUG_FLAG} login code.ornl.gov:4567 -u atj -p $(cat /home/builder/container-registry-token) +fi # Spin up local registry -docker run -d -p 5000:5000 --restart=always --name registry registry:2 +docker ${DEBUG_FLAG} run -d -p 5000:5000 --restart=always --name registry registry:2 # Build the Dockerfile docker image in the current directory -docker build -t localhost:5000/docker_image -f ./container.def +docker ${DEBUG_FLAG} build -t localhost:5000/docker_image -f ./container.def # Push to the local registry -docker push localhost:5000/docker_image +docker ${DEBUG_FLAG} push localhost:5000/docker_image # Build the singularity container from the docker image -singularity pull --name container.simg docker://localhost:5000/docker_image \ No newline at end of file +singularity ${DEBUG_FLAG} pull --name container.simg docker://localhost:5000/docker_image \ No newline at end of file diff --git a/Scripts/SingularityBuilderBackend b/Scripts/SingularityBuilderBackend index 664c7cc..afec989 100755 --- a/Scripts/SingularityBuilderBackend +++ b/Scripts/SingularityBuilderBackend @@ -1,8 +1,27 @@ #!/bin/bash -set -e +# Test for any arguments, such as --debug +for i in "$@" +do +case ${i} in + --debug) + DEBUG_FLAG='--debug' + shift # past argument with no value + ;; + *) + echo "unknown argument to SingularityBuilderBackend" + exit 1 + ;; +esac +done -# Provide readonly access to the private gitlab docker repository -docker login code.ornl.gov:4567 -u atj -p $(cat /home/builder/container-registry-token) +# Provide readonly access to the private gitlab docker repository if using the container-recipes docker registry +grep 'code.ornl.gov:4567' ./container.def +GREP_RC=$? +if [[ ${GREP_RC} -eq 0 ]] ; then + echo "Using container recipes docker registry login credentials" + export SINGULARITY_DOCKER_USERNAME=atj + export SINGULARITY_DOCKER_PASSWORD=$(cat /home/builder/container-registry-token) +fi -/usr/bin/unbuffer /usr/local/bin/singularity build ./container.simg ./container.def \ No newline at end of file +/usr/bin/unbuffer /usr/local/bin/singularity ${DEBUG_FLAG} build ./container.simg ./container.def \ No newline at end of file