diff --git a/cf-agent/verify_files_utils.c b/cf-agent/verify_files_utils.c index 22ee22a18a..541c68bd89 100644 --- a/cf-agent/verify_files_utils.c +++ b/cf-agent/verify_files_utils.c @@ -1551,7 +1551,7 @@ bool CopyRegularFile(EvalContext *ctx, const char *source, const char *dest, con return false; } - if (!CopyRegularFileNet(source, ToChangesPath(new), + if (!CopyRegularFileNet(source, dest, ToChangesPath(new), sstat->st_size, attr->copy.encrypt, conn, sstat->st_mode)) { RecordFailure(ctx, pp, attr, "Failed to copy file '%s' from '%s'", diff --git a/cf-serverd/server_common.c b/cf-serverd/server_common.c index 6322a1a9d2..91ba9265b2 100644 --- a/cf-serverd/server_common.c +++ b/cf-serverd/server_common.c @@ -54,6 +54,7 @@ static const int CF_NOSIZE = -1; #include /* ThreadLock */ #include /* struct Stat */ #include /* GetUserID() */ +#include #include "server_access.h" @@ -402,6 +403,9 @@ static void FailedTransfer(ConnectionInfo *connection) void CfGetFile(ServerFileGetState *args) { + assert(args != NULL); + assert(args->conn != NULL); + int fd; off_t n_read, total = 0, sendlen = 0, count = 0; char sendbuffer[CF_BUFSIZE + 256], filename[CF_BUFSIZE - 128]; @@ -421,12 +425,23 @@ void CfGetFile(ServerFileGetState *args) if (!TransferRights(args->conn, filename, &sb)) { + const ProtocolVersion version = ConnectionInfoProtocolVersion(conn_info); + assert(ProtocolIsKnown(version)); + Log(LOG_LEVEL_INFO, "REFUSE access to file: %s", filename); + + if (ProtocolSupportsFileStream(version)) { + Log(LOG_LEVEL_VERBOSE, "REFUSAL to user='%s' of request: %s", + NULL_OR_EMPTY(args->conn->username) ? "?" : args->conn->username, + args->replyfile); + FileStreamRefuse(args->conn->conn_info->ssl); + return; + } + /* Else then handle older protocols */ + RefuseAccess(args->conn, args->replyfile); snprintf(sendbuffer, CF_BUFSIZE, "%s", CF_FAILEDSTR); - const ProtocolVersion version = ConnectionInfoProtocolVersion(conn_info); - assert(ProtocolIsKnown(version)); if (ProtocolIsClassic(version)) { SendSocketStream(ConnectionInfoSocket(conn_info), sendbuffer, args->buf_size); @@ -440,6 +455,12 @@ void CfGetFile(ServerFileGetState *args) /* File transfer */ + const ProtocolVersion version = ConnectionInfoProtocolVersion(conn_info); + if (ProtocolSupportsFileStream(version)) { + FileStreamServe(conn_info->ssl, filename); + return; + } + if ((fd = safe_open(filename, O_RDONLY)) == -1) { Log(LOG_LEVEL_ERR, "Open error of file '%s'. (open: %s)", diff --git a/configure.ac b/configure.ac index 96609095da..6029ae0841 100644 --- a/configure.ac +++ b/configure.ac @@ -521,6 +521,21 @@ CF3_WITH_LIBRARY(pcre2, [ )] ) +dnl librsync + +AC_ARG_WITH([librsync], [AS_HELP_STRING([--with-librsync[[=PATH]]], [Spefify librsync path])], [], [with_librsync=yes]) + +if test "x$with_librsync" = "xno"; then + AC_MSG_ERROR([librsync is required]) +fi + +CF3_WITH_LIBRARY(librsync, [ + AC_CHECK_HEADERS([librsync.h], [], AC_MSG_ERROR(Cannot find librsync)) + AC_CHECK_HEADERS([librsync_export.h], [], AC_MSG_ERROR(Cannot find librsync)) + AC_CHECK_LIB(rsync, rs_file_size, [], [AC_MSG_ERROR(Cannot find librsync)]) + ] +) + dnl defined for libntech AC_DEFINE(WITH_PCRE2, 1, [Define if PCRE2 is being used]) @@ -1317,10 +1332,10 @@ dnl ###################################################################### dnl Collect all the options dnl ###################################################################### -CORE_CPPFLAGS="$LMDB_CPPFLAGS $TOKYOCABINET_CPPFLAGS $QDBM_CPPFLAGS $PCRE2_CPPFLAGS $OPENSSL_CPPFLAGS $SQLITE3_CPPFLAGS $LIBACL_CPPFLAGS $LIBCURL_CPPFLAGS $LIBYAML_CPPFLAGS $POSTGRESQL_CPPFLAGS $MYSQL_CPPFLAGS $LIBXML2_CPPFLAGS $CPPFLAGS $CFECOMPAT_CPPFLAGS" -CORE_CFLAGS="$LMDB_CFLAGS $TOKYOCABINET_CFLAGS $QDBM_CFLAGS $PCRE2_CFLAGS $OPENSSL_CFLAGS $SQLITE3_CFLAGS $LIBACL_CFLAGS $LIBCURL_CFLAGS $LIBYAML_CFLAGS $POSTGRESQL_CFLAGS $MYSQL_CFLAGS $LIBXML2_CFLAGS $CFLAGS" -CORE_LDFLAGS="$LMDB_LDFLAGS $TOKYOCABINET_LDFLAGS $QDBM_LDFLAGS $PCRE2_LDFLAGS $OPENSSL_LDFLAGS $SQLITE3_LDFLAGS $LIBACL_LDFLAGS $LIBCURL_LDFLAGS $LIBYAML_LDFLAGS $POSTGRESQL_LDFLAGS $MYSQL_LDFLAGS $LIBXML2_LDFLAGS $LDFLAGS" -CORE_LIBS="$LMDB_LIBS $TOKYOCABINET_LIBS $QDBM_LIBS $PCRE2_LIBS $OPENSSL_LIBS $SQLITE3_LIBS $LIBACL_LIBS $LIBCURL_LIBS $LIBYAML_LIBS $POSTGRESQL_LIBS $MYSQL_LIBS $LIBXML2_LIBS $LIBS" +CORE_CPPFLAGS="$LMDB_CPPFLAGS $TOKYOCABINET_CPPFLAGS $QDBM_CPPFLAGS $PCRE2_CPPFLAGS $OPENSSL_CPPFLAGS $SQLITE3_CPPFLAGS $LIBACL_CPPFLAGS $LIBCURL_CPPFLAGS $LIBRSYNC_CPPFLAGS $LIBYAML_CPPFLAGS $POSTGRESQL_CPPFLAGS $MYSQL_CPPFLAGS $LIBXML2_CPPFLAGS $CPPFLAGS $CFECOMPAT_CPPFLAGS" +CORE_CFLAGS="$LMDB_CFLAGS $TOKYOCABINET_CFLAGS $QDBM_CFLAGS $PCRE2_CFLAGS $OPENSSL_CFLAGS $SQLITE3_CFLAGS $LIBACL_CFLAGS $LIBCURL_CFLAGS $LIBRSYNC_CFLAGS $LIBYAML_CFLAGS $POSTGRESQL_CFLAGS $MYSQL_CFLAGS $LIBXML2_CFLAGS $CFLAGS" +CORE_LDFLAGS="$LMDB_LDFLAGS $TOKYOCABINET_LDFLAGS $QDBM_LDFLAGS $PCRE2_LDFLAGS $OPENSSL_LDFLAGS $SQLITE3_LDFLAGS $LIBACL_LDFLAGS $LIBCURL_LDFLAGS $LIBRSYNC_LDFLAGS $LIBYAML_LDFLAGS $POSTGRESQL_LDFLAGS $MYSQL_LDFLAGS $LIBXML2_LDFLAGS $LDFLAGS" +CORE_LIBS="$LMDB_LIBS $TOKYOCABINET_LIBS $QDBM_LIBS $PCRE2_LIBS $OPENSSL_LIBS $SQLITE3_LIBS $LIBACL_LIBS $LIBCURL_LIBS $LIBRSYNC_LIBS $LIBYAML_LIBS $POSTGRESQL_LIBS $MYSQL_LIBS $LIBXML2_LIBS $LIBS" dnl ###################################################################### dnl Make them available to subprojects. diff --git a/libcfnet/Makefile.am b/libcfnet/Makefile.am index 3e5cff8619..ddb5e50f1d 100644 --- a/libcfnet/Makefile.am +++ b/libcfnet/Makefile.am @@ -46,4 +46,5 @@ libcfnet_la_SOURCES = \ server_code.c server_code.h \ stat_cache.c stat_cache.h \ tls_client.c tls_client.h \ - tls_generic.c tls_generic.h + tls_generic.c tls_generic.h \ + file_stream.c file_stream.h diff --git a/libcfnet/README.md b/libcfnet/README.md index 568b920ef1..38e69b9e91 100644 --- a/libcfnet/README.md +++ b/libcfnet/README.md @@ -15,6 +15,7 @@ Names of protocol versions: 1. `"classic"` - Legacy, pre-TLS, protocol. Not enabled or allowed by default. 2. `"tls"` - TLS Protocol using OpenSSL. Encrypted and 2-way authentication. 3. `"cookie"` - TLS Protocol with cookie command for duplicate host detection. +3. `"filestream"` - Introduces a new streaming API for get file request (powered by librsync). Wanted protocol version can be specified from policy: @@ -59,3 +60,30 @@ Both server and client will then set `conn_info->protocol` to `2`, and use proto There is currently no way to require a specific version number (only allow / disallow version 1). This is because version 2 and 3 are practically identical. Downgrade from version 3 to 2 happens seamlessly, but crucially, it doesn't downgrade to version 1 inside the TLS code. + +## Commands + +### `GET ` (protocol v4) + +The following is a description of the `GET ` command, modified in +protocol version v4 (inroduced in CFEngine 3.25). + +The initial motivation for creating a new protocol version `"filestream"` was +due to a race condition found in the `GET ` request. It relied on the +file size aquired by `STAT `. However, if the file size increased +between the two requests, the client would think that the remaining data at the +offset of the aquired file size is a new protocol header. Which again would lead +to undefined behaviour. Hence, we needed a new protocol to send files. Instead +of reinventing the wheel, we decided to use librsync which utilizes the RSYNC +protocol to transmit files. + +The server implementation is found in function +[CfGet()](../cf-serverd/server_common.c). Client impementations are found in +[CopyRegularFileNet()](client_code.c) and [ProtocolGet()](protocol.c) + +Similar to before, the client issues a `GET ` request. However, +instead of continuing to execute the old protocol, the client immediately calls +`FileStreamFetch()` from the "File Stream API". Upon receiving such a request, +the server calls either `FileStreamRefuse()` (to refuse the request) or +`FileStreamServe()` (to comply with the request). The internal workings of the +File Stream API is well explained in [file_stream.h](file_stream.h). diff --git a/libcfnet/client_code.c b/libcfnet/client_code.c index 7dccb88324..0d1dff506b 100644 --- a/libcfnet/client_code.c +++ b/libcfnet/client_code.c @@ -45,6 +45,7 @@ #include /* ProgrammingError */ #include /* PRINTSIZE */ #include /* LastSaw */ +#include #define CFENGINE_SERVICE "cfengine" @@ -749,9 +750,11 @@ static void FlushFileStream(int sd, int toget) /* TODO finalise socket or TLS session in all cases that this function fails * and the transaction protocol is out of sync. */ -bool CopyRegularFileNet(const char *source, const char *dest, off_t size, +bool CopyRegularFileNet(const char *source, const char *basis, const char *dest, off_t size, bool encrypt, AgentConnection *conn, mode_t mode) { + assert(conn != NULL); + char *buf, workbuf[CF_BUFSIZE], cfchangedstr[265]; const int buf_size = 2048; @@ -774,23 +777,12 @@ bool CopyRegularFileNet(const char *source, const char *dest, off_t size, unlink(dest); /* To avoid link attacks */ - int dd = safe_open_create_perms(dest, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL | O_BINARY, mode); - if (dd == -1) - { - Log(LOG_LEVEL_ERR, - "Copy from server '%s' to destination '%s' failed (open: %s)", - conn->this_server, dest, GetErrorStr()); - unlink(dest); - return false; - } - workbuf[0] = '\0'; int tosend = snprintf(workbuf, CF_BUFSIZE, "GET %d %s", buf_size, source); if (tosend <= 0 || tosend >= CF_BUFSIZE) { Log(LOG_LEVEL_ERR, "Failed to compose GET command for file %s", source); - close(dd); return false; } @@ -799,7 +791,21 @@ bool CopyRegularFileNet(const char *source, const char *dest, off_t size, if (SendTransaction(conn->conn_info, workbuf, tosend, CF_DONE) == -1) { Log(LOG_LEVEL_ERR, "Couldn't send GET command"); - close(dd); + return false; + } + + const ProtocolVersion version = ConnectionInfoProtocolVersion(conn->conn_info); + if (ProtocolSupportsFileStream(version)) { + return FileStreamFetch(conn->conn_info->ssl, basis, dest, mode); + } + + int dd = safe_open_create_perms(dest, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL | O_BINARY, mode); + if (dd == -1) + { + Log(LOG_LEVEL_ERR, + "Copy from server '%s' to destination '%s' failed (open: %s)", + conn->this_server, dest, GetErrorStr()); + unlink(dest); return false; } diff --git a/libcfnet/client_code.h b/libcfnet/client_code.h index 4aaaa5ff4c..8f9a7cef57 100644 --- a/libcfnet/client_code.h +++ b/libcfnet/client_code.h @@ -47,7 +47,7 @@ AgentConnection *ServerConnection(const char *server, const char *port, const Rl void DisconnectServer(AgentConnection *conn); bool CompareHashNet(const char *file1, const char *file2, bool encrypt, AgentConnection *conn); -bool CopyRegularFileNet(const char *source, const char *dest, off_t size, +bool CopyRegularFileNet(const char *source, const char *basis, const char *dest, off_t size, bool encrypt, AgentConnection *conn, mode_t mode); Item *RemoteDirList(const char *dirname, bool encrypt, AgentConnection *conn); diff --git a/libcfnet/file_stream.c b/libcfnet/file_stream.c new file mode 100644 index 0000000000..396618db46 --- /dev/null +++ b/libcfnet/file_stream.c @@ -0,0 +1,913 @@ +/* + Copyright 2024 Northern.tech AS + + This file is part of CFEngine 3 - written and maintained by Northern.tech AS. + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + + To the extent this program is licensed as part of the Enterprise + versions of CFEngine, the applicable Commercial Open Source License + (COSL) may apply to this file if you as a licensee so wish it. See + included file COSL.txt. +*/ + +#include + +#include "file_stream.h" + +#include +#include +#include +#include +#include +#include + +/*********************************************************/ +/* Network protocol */ +/*********************************************************/ + +/** + * @brief Simple network protocol on top of SSL/TCP. Used for client-server + * communication during RSYNC stream. + * + * @details Header format: + * +----------+----------+----------+----------+ + * | SDU Len. | Reserved | EOF Flag | ERR Flag | + * +----------+----------+----------+----------+ + * | 12 bits | 2 bits | 1 bit | 1 bit | + * +----------+----------+----------+----------+ + * + * The header fields are defined as follows: + * SDU Length Length of the SDU (i.e. payload) encapsulated within + * this datagram. + * Reserved 2 bits reserved for future use. + * End-of-File flag Signals whether or not the receiver should expect to + * receive more datagrams. + * Error flag Signals that the transmission must be canceled due to + * unexpected error. + * + * @note If the End-of-File flag is set, there may still be data to process in + * in the payload. If the Error flag is set, there may be an error + * message in the payload. + */ +#define HEADER_SIZE ((int) sizeof(uint16_t)) + +/** + * @note The TLS Generic API requires that the message length is less than + * CF_BUFSIZE. Furthermore, the protocol can only handle up to 4095 + * Bytes, because it's the largest unsigned integer you can represent + * with 12 bits (2^12 - 1 = 4095). + */ +#define MESSAGE_SIZE MIN(CF_BUFSIZE - 1, 4095) + +/** + * @brief Send a message using the file stream protocol + * @warning You probably want to use SendMessage() or SendError() instead + * + * @param conn The SSL connection object + * @param msg The message to send + * @param len The length of the message to send (must be less or equal to + * MESSAGE_SIZE Bytes) + * @param eof Set to true if this is the last message in a transaction, + * otherwise false + * @param err Set to true if transaction must be canceled (e.g., due to an + * unexpected error), otherwise false + * @return true on success, otherwise false + */ +static bool __SendMessage( + SSL *conn, const char *msg, size_t len, bool eof, bool err) +{ + assert(conn != NULL); + assert(msg != NULL || len == 0); + assert(len <= MESSAGE_SIZE); + + /* Set message length */ + uint16_t header = len << 4; + + /* Set Error flag */ + if (err) + { + header |= (1 << 0); + } + + /* Set End-of-File flag */ + if (eof) + { + header |= (1 << 1); + } + + /* Send header */ + header = htons(header); + int ret = TLSSend(conn, (char *) &header, HEADER_SIZE); + if (ret != HEADER_SIZE) + { + Log(LOG_LEVEL_ERR, + "Failed to send message header during file stream (%d != %d)", + ret, + HEADER_SIZE); + return false; + } + + if (len > 0) + { + /* Send payload */ + ret = TLSSend(conn, msg, len); + if (ret != (int) len) + { + Log(LOG_LEVEL_ERR, + "Failed to send message payload during file stream (%d != %zu)", + ret, + len); + return false; + } + } + + return true; +} + +/** + * @brief Send a message using the file stream protocol + * + * @param conn The SSL connection object + * @param msg The message to send + * @param len The length of the message to send (must be less or equal to + * MESSAGE_SIZE Bytes) + * @param eof Set to true if this is the last message in a transaction, + * otherwise false + * @return true on success, otherwise false + */ +static inline bool SendMessage( + SSL *conn, const char *msg, size_t len, bool eof) +{ + assert(conn != NULL); + assert(msg != NULL || len == 0); + + return __SendMessage(conn, msg, len, eof, false); +} + +/** + * @brief Receive a message using the file stream protocol + * + * @param conn The SSL connection object + * @param msg The message receive buffer (must be MESSAGE_SIZE Bytes large) + * @param len The length of the reveived message + * @param eof Is set to true if this was the last message in the transaction + * @return true on success, otherwise false + * + * @note RecvMessage fails if the communication is broken or if we received an + * error from the remote host. In both cases, we should not try to flush + * the stream. + */ +static bool RecvMessage(SSL *conn, char *msg, size_t *len, bool *eof) +{ + assert(conn != NULL); + assert(msg != NULL); + assert(len != NULL); + assert(eof != NULL); + + /* TLSRecv() expects a buffer this size */ + char recv_buffer[CF_BUFSIZE]; + + /* Receive header */ + uint16_t header; + int ret = TLSRecv(conn, recv_buffer, HEADER_SIZE); + if (ret != HEADER_SIZE) + { + Log(LOG_LEVEL_ERR, + "Failed to receive message header during file stream (%d != %d)", + ret, + HEADER_SIZE); + return false; + } + memcpy(&header, recv_buffer, HEADER_SIZE); + header = ntohs(header); + + /* Extract Error flag */ + bool err = header & (1 << 0); + + /* Extract End-of-File flag */ + *eof = header & (1 << 1); + + /* Extract message length */ + *len = header >> 4; + + /* Read payload */ + if (*len > 0) + { + ret = TLSRecv(conn, recv_buffer, *len); + if (ret != *len) + { + Log(LOG_LEVEL_ERR, + "Failed to receive message payload during file stream (%d != %zu)", + ret, + *len); + return false; + } + memcpy(msg, recv_buffer, *len); + + if (err) + { + /* If the error flag is set, then the payload contains an error + * message in the form of a NUL-byte terminated string. */ + Log(LOG_LEVEL_ERR, "Remote file stream error: %s", msg); + } + } + + return !err; +} + +/** + * @brief Flush the file stream + * + * It's used to prevent the remote host from blocking while sending the + * remaining data after we have experienced an unexpected error and need to + * abort the file stream. Once the stream has been successfully flushed, the + * remote host will be ready to receive our error message. + * + * @param conn The SSL connection object + * @return true on success, otherwise false + */ +static bool FlushStream(SSL *conn) +{ + assert(conn != NULL); + + char msg[MESSAGE_SIZE]; + size_t len; + bool eof; + while (RecvMessage(conn, msg, &len, &eof)) + { + if (eof) + { + return true; + } + } + + Log(LOG_LEVEL_ERR, "Remote file stream error: %s", msg); + return false; +} + +/** + * @brief Send an error message using the file stream protocol + * + * @param conn The SSL connection object + * @param flush Whether or not to flush the stream (see FlushStream()) + * @param fmt The format string + * @param ... The format string arguments + * @return true on success, otherwise false + */ +static bool SendError(SSL *conn, bool flush, const char *fmt, ...) +{ + assert(conn != NULL); + assert(fmt != NULL); + + va_list ap; + char msg[MESSAGE_SIZE]; + + va_start(ap, fmt); + int len = vsnprintf(msg, MESSAGE_SIZE, fmt, ap); + va_end(ap); + + assert(len >= 0); /* Let's make sure we detect this in debug builds */ + if (len < 0) + { + Log(LOG_LEVEL_ERR, + "Failed to format error message during file stream"); + len = 0; /* We still want to send the header */ + } + else if (len >= MESSAGE_SIZE) + { + Log(LOG_LEVEL_WARNING, + "Error message truncated during file stream (%d >= %d)", + len, + MESSAGE_SIZE); + /* Add ... to indicate message truncation */ + msg[MESSAGE_SIZE - 1] = '.'; + msg[MESSAGE_SIZE - 2] = '.'; + msg[MESSAGE_SIZE - 3] = '.'; + len = MESSAGE_SIZE; + } + + if (flush) + { + FlushStream(conn); + } + + return __SendMessage(conn, msg, (size_t) len, false, true); +} + +/*********************************************************/ +/* Server specific */ +/*********************************************************/ + +#define ERROR_MSG_UNSPECIFIED_SERVER_REFUSAL "Unspecified server refusal" +#define ERROR_MSG_INTERNAL_SERVER_ERROR "Internal server error" + +bool FileStreamRefuse(SSL *conn) +{ + return SendError(conn, false, ERROR_MSG_UNSPECIFIED_SERVER_REFUSAL); +} + +/** + * @brief Receive and load signature into memory + * + * @param conn The SSL connection object + * @param sig The signature of the outdated file + * @return true on success, otherwise false + */ +static bool RecvSignature(SSL *conn, rs_signature_t **sig) +{ + assert(conn != NULL); + assert(sig != NULL); + + /* The input buffer has to be twice the message size, so that if can fit a + * new message, as well as some tail data from the last job iteration */ + char in_buf[MESSAGE_SIZE * 2]; + + /* Start a job for loading a signature into memory */ + rs_job_t *job = rs_loadsig_begin(sig); + if (job == NULL) + { + SendError(conn, true, ERROR_MSG_INTERNAL_SERVER_ERROR); + return false; + } + + /* Setup buffers for the job */ + rs_buffers_t bufs = {0}; + + rs_result res; + do + { + /* Fill input buffers */ + if (bufs.eof_in == 0) + { + if (bufs.avail_in > MESSAGE_SIZE) + { + /* The job requires more data, but we cannot fit another + * message into the input buffer */ + Log(LOG_LEVEL_ERR, + "Insufficient buffer capacity to receive file stream signature"); + SendError(conn, true, ERROR_MSG_INTERNAL_SERVER_ERROR); + + rs_job_free(job); + return false; + } + + if (bufs.avail_in > 0) + { + /* Move leftover tail data to the front of the buffer */ + memmove(in_buf, bufs.next_in, bufs.avail_in); + } + + size_t n_bytes; + bool eof; + if (!RecvMessage(conn, in_buf + bufs.avail_in, &n_bytes, &eof)) + { + /* Error is already logged */ + rs_job_free(job); + return false; + } + + bufs.eof_in = eof ? 1 : 0; + bufs.next_in = in_buf; + bufs.avail_in += n_bytes; + } + + /* Iterate job */ + res = rs_job_iter(job, &bufs); + if (res != RS_DONE && res != RS_BLOCKED) + { + SendError(conn, bufs.eof_in == 0, ERROR_MSG_INTERNAL_SERVER_ERROR); + rs_job_free(job); + return false; + } + } while (res != RS_DONE); + + rs_job_free(job); + + return true; +} + +/** + * @brief Compute and send delta based on the source file and the signature of + * the basis file + * + * @param conn The SSL connection object + * @param sig The signature of the basis file + * @param filename The name of the source file + * @return true on success, otherwise false + */ +static bool SendDelta(SSL *conn, rs_signature_t *sig, const char *filename) +{ + assert(conn != NULL); + assert(sig != NULL); + assert(filename != NULL); + + /* In this case, the input buffer does not need to be twice the message + * size, because we can control how much we read into it */ + char in_buf[MESSAGE_SIZE], out_buf[MESSAGE_SIZE]; + + /* Open source file */ + FILE *file = safe_fopen(filename, "rb"); + if (file == NULL) + { + Log(LOG_LEVEL_ERR, + "Failed to open the source file '%s' for computing delta during file stream: %s", + filename, + GetErrorStr()); + SendError(conn, false, ERROR_MSG_UNSPECIFIED_SERVER_REFUSAL); + return false; + } + + /* Build hash table */ + rs_result res = rs_build_hash_table(sig); + if (res != RS_DONE) + { + SendError(conn, false, ERROR_MSG_INTERNAL_SERVER_ERROR); + fclose(file); + return false; + } + + /* Start generating delta */ + rs_job_t *job = rs_delta_begin(sig); + if (job == NULL) + { + SendError(conn, false, ERROR_MSG_INTERNAL_SERVER_ERROR); + fclose(file); + return false; + } + + /* Setup buffers for the job */ + rs_buffers_t bufs = {0}; + bufs.next_out = out_buf; + bufs.avail_out = MESSAGE_SIZE; /* We cannot send more using the protocol */ + + do + { + /* Fill input buffers */ + if (bufs.eof_in == 0) + { + if (bufs.avail_in >= MESSAGE_SIZE) + { + /* The job requires more data, but the input buffer is full */ + Log(LOG_LEVEL_ERR, + "Insufficient buffer capacity to compute delta"); + SendError(conn, false, ERROR_MSG_INTERNAL_SERVER_ERROR); + + fclose(file); + rs_job_free(job); + return false; + } + + if (bufs.avail_in > 0) + { + /* Move leftover tail data to the front of the buffer */ + memmove(in_buf, bufs.next_in, bufs.avail_in); + } + + size_t n_bytes = fread( + in_buf + bufs.avail_in, + 1 /* Byte */, + sizeof(in_buf) - bufs.avail_in, + file); + if (n_bytes == 0) + { + if (ferror(file)) + { + Log(LOG_LEVEL_ERR, + "Failed to read the source file '%s' during file stream: %s", + filename, + GetErrorStr()); + SendError(conn, false, ERROR_MSG_INTERNAL_SERVER_ERROR); + + fclose(file); + rs_job_free(job); + return false; + } + + /* End-of-File reached */ + bufs.eof_in = feof(file); + assert(bufs.eof_in != 0); + } + + bufs.next_in = in_buf; + bufs.avail_in += n_bytes; + } + + /* Iterate job */ + res = rs_job_iter(job, &bufs); + if (res != RS_DONE && res != RS_BLOCKED) + { + SendError(conn, false, ERROR_MSG_INTERNAL_SERVER_ERROR); + + fclose(file); + rs_job_free(job); + return false; + } + + /* Drain output buffer, if there is data */ + size_t present = bufs.next_out - out_buf; + if (present > 0) + { + assert(present <= MESSAGE_SIZE); + if (!SendMessage(conn, out_buf, present, res == RS_DONE)) + { + fclose(file); + rs_job_free(job); + return false; + } + + bufs.next_out = out_buf; + bufs.avail_out = MESSAGE_SIZE; + } + else if (res == RS_DONE) + { + /* Send End-of-File */ + if (!SendMessage(conn, NULL, 0, 1)) + { + fclose(file); + rs_job_free(job); + return false; + } + } + } while (res != RS_DONE); + + fclose(file); + rs_job_free(job); + + return true; +} + +bool FileStreamServe(SSL *conn, const char *filename) +{ + assert(conn != NULL); + assert(filename != NULL); + + Log(LOG_LEVEL_VERBOSE, + "Receiving- & loading signature into memory for file '%s'...", + filename); + rs_signature_t *sig; + if (!RecvSignature(conn, &sig)) + { + /* Error is already logged */ + return false; + } + + Log(LOG_LEVEL_VERBOSE, + "Computing- & sending delta for file '%s'...", + filename); + if (!SendDelta(conn, sig, filename)) + { + /* Error is already logged */ + rs_free_sumset(sig); + return false; + } + + rs_free_sumset(sig); + return true; +} + +/*********************************************************/ +/* Client specific */ +/*********************************************************/ + +#define ERROR_MSG_INTERNAL_CLIENT_ERROR "Internal client error" + +/** + * @brief Compute and send a signature of the basis file + * + * @param conn The SSL connection object + * @param filename The name of the basis file + * @return true on success, otherwise false + */ +static bool SendSignature(SSL *conn, const char *filename) +{ + assert(conn != NULL); + assert(filename != NULL); + + /* In this case, the input buffer does not need to be twice the message + * size, because we can control how much we read into it */ + char in_buf[MESSAGE_SIZE], out_buf[MESSAGE_SIZE]; + + /* Open basis file */ + FILE *file = safe_fopen(filename, "rb"); + if (file == NULL) + { + Log(LOG_LEVEL_ERR, + "Failed to open the basis file '%s' for computing delta during file stream: %s", + filename, + GetErrorStr()); + SendError(conn, false, ERROR_MSG_INTERNAL_CLIENT_ERROR); + return false; + } + + /* Get file size */ + rs_long_t fsize = rs_file_size(file); + + /* Get recommended arguments */ + rs_magic_number sig_magic = 0; + size_t block_len = 0, strong_len = 0; + rs_result res = rs_sig_args(fsize, &sig_magic, &block_len, &strong_len); + if (res != RS_DONE) + { + SendError(conn, false, ERROR_MSG_INTERNAL_CLIENT_ERROR); + fclose(file); + return false; + } + + /* Start generating signature */ + rs_job_t *job = rs_sig_begin(block_len, strong_len, sig_magic); + if (job == NULL) + { + SendError(conn, false, ERROR_MSG_INTERNAL_CLIENT_ERROR); + fclose(file); + return false; + } + + /* Setup buffers */ + rs_buffers_t bufs = {0}; + bufs.next_out = out_buf; + bufs.avail_out = MESSAGE_SIZE; /* We cannot send more using the protocol */ + + do + { + if (bufs.eof_in == 0) + { + if (bufs.avail_in >= MESSAGE_SIZE) + { + /* The job requires more data, but the input buffer is full */ + Log(LOG_LEVEL_ERR, + "Insufficient buffer capacity to compute delta"); + SendError(conn, false, ERROR_MSG_INTERNAL_CLIENT_ERROR); + + fclose(file); + rs_job_free(job); + return false; + } + + if (bufs.avail_in > 0) + { + /* Move leftover tail data to the front of the buffer */ + memmove(in_buf, bufs.next_in, bufs.avail_in); + } + + /* Fill input buffer */ + size_t n_bytes = fread( + in_buf + bufs.avail_in, + 1 /* Byte */, + sizeof(in_buf) - bufs.avail_in, + file); + if (n_bytes == 0) + { + if (ferror(file)) + { + Log(LOG_LEVEL_ERR, + "Failed to read the basis file '%s' during file stream: %s", + filename, + GetErrorStr()); + SendError(conn, false, ERROR_MSG_INTERNAL_CLIENT_ERROR); + + fclose(file); + rs_job_free(job); + return false; + } + + /* End-of-File reached */ + bufs.eof_in = feof(file); + assert(bufs.eof_in != 0); + } + + bufs.next_in = in_buf; + bufs.avail_in += n_bytes; + } + + /* Iterate job */ + res = rs_job_iter(job, &bufs); + if (res != RS_DONE && res != RS_BLOCKED) + { + SendError(conn, false, ERROR_MSG_INTERNAL_CLIENT_ERROR); + + fclose(file); + rs_job_free(job); + return false; + } + + /* Drain output buffer, if there is data */ + size_t present = bufs.next_out - out_buf; + if (present > 0) + { + assert(present <= MESSAGE_SIZE); + if (!SendMessage(conn, out_buf, present, res == RS_DONE)) + { + fclose(file); + rs_job_free(job); + return false; + } + + bufs.next_out = out_buf; + bufs.avail_out = MESSAGE_SIZE; + } + else if (res == RS_DONE) + { + /* Send End-of-File */ + if (!SendMessage(conn, NULL, 0, 1)) + { + fclose(file); + rs_job_free(job); + return false; + } + } + } while (res != RS_DONE); + + fclose(file); + rs_job_free(job); + + return true; +} + +/** + * @brief Receive delta and apply patch to the outdated copy of the file + * + * @param conn The SSL connection object + * @param basis The name of basis file + * @param dest The name of destination file + * @param perms The desired file permissions of the destination file + * @return true on success, otherwise false + */ +static bool RecvDelta( + SSL *conn, const char *basis, const char *dest, mode_t perms) +{ + assert(conn != NULL); + assert(basis != NULL); + assert(dest != NULL); + + /* The input buffer has to be twice the message size, so that if can fit a + * new message, as well as some tail data from the last job iteration */ + char in_buf[MESSAGE_SIZE * 2], out_buf[MESSAGE_SIZE]; + + /* Open/create the destination file */ + FILE *new = safe_fopen_create_perms(dest, "wb", perms); + if (new == NULL) + { + Log(LOG_LEVEL_ERR, + "Failed to open/create destination file '%s': %s", + dest, + GetErrorStr()); + SendError(conn, true, ERROR_MSG_INTERNAL_CLIENT_ERROR); + return false; + } + + /* Open the basis file */ + FILE *old = safe_fopen(basis, "rb"); + if (old == NULL) + { + Log(LOG_LEVEL_ERR, + "Failed to open basis file '%s': %s", + basis, + GetErrorStr()); + SendError(conn, true, ERROR_MSG_INTERNAL_CLIENT_ERROR); + return false; + } + + /* Start a job for patching destination file */ + rs_job_t *job = rs_patch_begin(rs_file_copy_cb, old); + if (job == NULL) + { + SendError(conn, true, ERROR_MSG_INTERNAL_CLIENT_ERROR); + return false; + } + + /* Setup buffers for the job */ + rs_buffers_t bufs = {0}; + bufs.next_out = out_buf; + bufs.avail_out = sizeof(out_buf); + + rs_result res; + do + { + /* Fill input buffers */ + if (bufs.eof_in == 0) + { + if (bufs.avail_in > MESSAGE_SIZE) + { + /* The job requires more data, but we cannot fit another + * message into the input buffer */ + Log(LOG_LEVEL_ERR, + "Insufficient buffer capacity to receive file stream delta"); + SendError(conn, true, ERROR_MSG_INTERNAL_CLIENT_ERROR); + + fclose(new); + fclose(old); + rs_job_free(job); + return false; + } + + if (bufs.avail_in > 0) + { + /* Move leftover tail data to the front of the buffer */ + memmove(in_buf, bufs.next_in, bufs.avail_in); + } + + size_t n_bytes; + bool eof; + if (!RecvMessage(conn, in_buf + bufs.avail_in, &n_bytes, &eof)) + { + /* Error is already logged */ + fclose(new); + fclose(old); + rs_job_free(job); + return false; + } + + bufs.eof_in = eof ? 1 : 0; + bufs.next_in = in_buf; + bufs.avail_in += n_bytes; + } + + res = rs_job_iter(job, &bufs); + if (res != RS_DONE && res != RS_BLOCKED) + { + SendError(conn, bufs.eof_in == 0, ERROR_MSG_INTERNAL_CLIENT_ERROR); + + fclose(new); + fclose(old); + rs_job_free(job); + return false; + } + + /* Drain output buffer, if there is data */ + size_t present = bufs.next_out - out_buf; + if (present > 0) + { + size_t n_bytes = fwrite(out_buf, 1 /* Byte */, present, new); + if (n_bytes == 0) + { + Log(LOG_LEVEL_ERR, + "Failed to write to destination file '%s' during file stream: %s", + dest, + GetErrorStr()); + SendError( + conn, bufs.eof_in == 0, ERROR_MSG_INTERNAL_CLIENT_ERROR); + + fclose(new); + fclose(old); + rs_job_free(job); + return false; + } + + bufs.next_out = out_buf; + bufs.avail_out = sizeof(out_buf); + } + } while (res != RS_DONE); + + fclose(new); + fclose(old); + rs_job_free(job); + + return true; +} + +bool FileStreamFetch( + SSL *conn, const char *basis, const char *dest, mode_t perms) +{ + assert(conn != NULL); + assert(basis != NULL); + assert(dest != NULL); + + /* Let's make sure the basis file exists */ + FILE *file = safe_fopen_create_perms(basis, "wb", perms); + if (file != NULL) + { + fclose(file); + } + + Log(LOG_LEVEL_VERBOSE, + "Computing- & sending signature of file '%s'...", + basis); + if (!SendSignature(conn, basis)) + { + /* Error is already logged */ + return false; + } + + Log(LOG_LEVEL_VERBOSE, + "Receiving delta & applying patch to file '%s'...", + dest); + if (!RecvDelta(conn, basis, dest, perms)) + { + /* Error is already logged */ + return false; + } + + return true; +} diff --git a/libcfnet/file_stream.h b/libcfnet/file_stream.h new file mode 100644 index 0000000000..736b5081a6 --- /dev/null +++ b/libcfnet/file_stream.h @@ -0,0 +1,98 @@ +/* + Copyright 2024 Northern.tech AS + + This file is part of CFEngine 3 - written and maintained by Northern.tech AS. + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + + To the extent this program is licensed as part of the Enterprise + versions of CFEngine, the applicable Commercial Open Source License + (COSL) may apply to this file if you as a licensee so wish it. See + included file COSL.txt. +*/ + +#ifndef FILE_STREAM_H +#define FILE_STREAM_H + +/** + * @file file_stream.h + * + * +---------+ +---------+ + * | client | | server | + * +----+----+ +----+----+ + * | | + * | sig = Compute(basis) | + * | | + * | Send(sig) ---------------->| sig = Recv() + * | | + * | | sig = BuildHashTable(sig) + * | | + * | | delta = Compute(sig, src) + * | | + * | delta = Recv() <-----------| Send(delta) + * | | + * | dest = Patch(delta, basis) | + * | | + * v v + * + * 1. Client generates a signature of the basis file (i.e., the "outdated" + * file) + * 2. Client sends signature to server + * 4. Server builds a hash table from the signature + * 5. Server generates delta from signature and the source file (i.e., the + * "up-to-date" file) + * 6. Server sends delta to client + * 7. Client applies delta on contents of the basis file in order to create + * the destination file + * + */ + +#include +#include +#include /* mode_t */ + + +/** + * @brief Reply with unspecified server refusal + * + * E.g., use this function to when the resource does not exist or access is + * denied. We don't disinguish between these two for security reasons. + * + * @param conn The SSL connection object + * @return true on success, otherwise false + */ +bool FileStreamRefuse(SSL *conn); + +/** + * @brief Serve a file using the stream API + * + * @param conn The SSL connection object + * @param filename The name of the source file + * @return true on success, otherwise false + */ +bool FileStreamServe(SSL *conn, const char *filename); + +/** + * @brief Fetch a file using the stream API + * + * @param conn The SSL connection object + * @param basis The name of the basis file + * @param dest The name of the destination file + * @param perms The desired permissions of the destination file + * @return true on success, otherwise false + */ +bool FileStreamFetch( + SSL *conn, const char *basis, const char *dest, mode_t perms); + +#endif // FILE_STREAM_H diff --git a/libcfnet/protocol.c b/libcfnet/protocol.c index 3a6f9bcb37..af03deea83 100644 --- a/libcfnet/protocol.c +++ b/libcfnet/protocol.c @@ -31,6 +31,7 @@ #include #include #include +#include Seq *ProtocolOpenDir(AgentConnection *conn, const char *path) { @@ -124,6 +125,37 @@ bool ProtocolGet(AgentConnection *conn, const char *remote_path, return false; } + /* Use file stream API if it is available */ + const ProtocolVersion version = ConnectionInfoProtocolVersion(conn->conn_info); + if (ProtocolSupportsFileStream(version)) + { + fclose(file_ptr); + + char dest[PATH_MAX]; + ret = snprintf(dest, sizeof(dest), "%s.cfnew", local_path); + if (ret < 0 || (size_t)ret >= sizeof(dest)) + { + Log(LOG_LEVEL_ERR, "Truncation error: Path too long (%d >= %zu)", ret, sizeof(dest)); + return false; + } + + if (!FileStreamFetch(conn->conn_info->ssl, local_path, dest, perms)) + { + /* Error is already logged */ + return false; + } + + if (rename(dest, local_path) == -1) + { + Log(LOG_LEVEL_ERR, "Failed to replace destination file '%s' with basis file '%s': %s", dest, local_path, GetErrorStr()); + return false; + } + + return true; + } + + /* Otherwise, use old protocol */ + char cfchangedstr[sizeof(CF_CHANGEDSTR1 CF_CHANGEDSTR2)]; snprintf(cfchangedstr, sizeof(cfchangedstr), "%s%s", CF_CHANGEDSTR1, CF_CHANGEDSTR2); diff --git a/libcfnet/protocol_version.c b/libcfnet/protocol_version.c index 31925e9a96..e99ef2c89d 100644 --- a/libcfnet/protocol_version.c +++ b/libcfnet/protocol_version.c @@ -33,6 +33,10 @@ ProtocolVersion ParseProtocolVersionPolicy(const char *const s) { return CF_PROTOCOL_COOKIE; } + else if (StringEqual(s, "4") || StringEqual(s, "filestream")) + { + return CF_PROTOCOL_FILESTREAM; + } else if (StringEqual(s, "latest")) { return CF_PROTOCOL_LATEST; diff --git a/libcfnet/protocol_version.h b/libcfnet/protocol_version.h index 06f7c32382..61e9cf3855 100644 --- a/libcfnet/protocol_version.h +++ b/libcfnet/protocol_version.h @@ -39,10 +39,11 @@ typedef enum /* --- Greater versions use TLS as secure communications layer --- */ CF_PROTOCOL_TLS = 2, CF_PROTOCOL_COOKIE = 3, + CF_PROTOCOL_FILESTREAM = 4, } ProtocolVersion; /* We use CF_PROTOCOL_LATEST as the default for new connections. */ -#define CF_PROTOCOL_LATEST CF_PROTOCOL_COOKIE +#define CF_PROTOCOL_LATEST CF_PROTOCOL_FILESTREAM static inline const char *ProtocolVersionString(const ProtocolVersion p) { @@ -54,6 +55,8 @@ static inline const char *ProtocolVersionString(const ProtocolVersion p) return "tls"; case CF_PROTOCOL_CLASSIC: return "classic"; + case CF_PROTOCOL_FILESTREAM: + return "filestream"; default: return "undefined"; } @@ -84,6 +87,11 @@ static inline bool ProtocolIsClassic(const ProtocolVersion p) return (p == CF_PROTOCOL_CLASSIC); } +static inline bool ProtocolSupportsFileStream(const ProtocolVersion p) +{ + return (p >= CF_PROTOCOL_FILESTREAM); +} + static inline bool ProtocolTerminateCSV(const ProtocolVersion p) { return (p < CF_PROTOCOL_COOKIE); diff --git a/libpromises/Makefile.am b/libpromises/Makefile.am index 4162972d18..b2123440de 100644 --- a/libpromises/Makefile.am +++ b/libpromises/Makefile.am @@ -30,7 +30,7 @@ AM_LDFLAGS = endif AM_LDFLAGS += $(CORE_LDFLAGS) $(LMDB_LDFLAGS) $(TOKYOCABINET_LDFLAGS) $(QDBM_LDFLAGS) \ - $(PCRE2_LDFLAGS) $(OPENSSL_LDFLAGS) $(SQLITE3_LDFLAGS) $(LIBACL_LDFLAGS) $(LIBYAML_LDFLAGS) $(LIBCURL_LDFLAGS) + $(PCRE2_LDFLAGS) $(OPENSSL_LDFLAGS) $(SQLITE3_LDFLAGS) $(LIBACL_LDFLAGS) $(LIBYAML_LDFLAGS) $(LIBCURL_LDFLAGS) $(LIBRSYNC_LDFLAGS) AM_CPPFLAGS = \ -I$(srcdir)/../libntech/libutils -I$(srcdir)/../libcfecompat \ @@ -39,16 +39,16 @@ AM_CPPFLAGS = \ -I$(srcdir)/../cf-check \ $(CORE_CPPFLAGS) $(ENTERPRISE_CPPFLAGS) \ $(LMDB_CPPFLAGS) $(TOKYOCABINET_CPPFLAGS) $(QDBM_CPPFLAGS) \ - $(PCRE2_CPPFLAGS) $(OPENSSL_CPPFLAGS) $(SQLITE3_CPPFLAGS) $(LIBACL_CPPFLAGS) $(LIBYAML_CPPFLAGS) $(LIBCURL_CPPFLAGS) + $(PCRE2_CPPFLAGS) $(OPENSSL_CPPFLAGS) $(SQLITE3_CPPFLAGS) $(LIBACL_CPPFLAGS) $(LIBYAML_CPPFLAGS) $(LIBCURL_CPPFLAGS) $(LIBRSYNC_CPPFLAGS) AM_CFLAGS = $(CORE_CFLAGS) $(ENTERPRISE_CFLAGS) \ $(LMDB_CFLAGS) $(TOKYOCABINET_CFLAGS) $(QDBM_CFLAGS) \ - $(PCRE2_CFLAGS) $(OPENSSL_CFLAGS) $(SQLITE3_CFLAGS) $(LIBACL_CFLAGS) $(LIBYAML_CFLAGS) $(LIBCURL_CFLAGS) + $(PCRE2_CFLAGS) $(OPENSSL_CFLAGS) $(SQLITE3_CFLAGS) $(LIBACL_CFLAGS) $(LIBYAML_CFLAGS) $(LIBCURL_CFLAGS) $(LIBRSYNC_CFLAGS) AM_YFLAGS = -d LIBS = $(LMDB_LIBS) $(TOKYOCABINET_LIBS) $(QDBM_LIBS) \ - $(PCRE2_LIBS) $(OPENSSL_LIBS) $(SQLITE3_LIBS) $(LIBACL_LIBS) $(LIBYAML_LIBS) $(LIBCURL_LIBS) + $(PCRE2_LIBS) $(OPENSSL_LIBS) $(SQLITE3_LIBS) $(LIBACL_LIBS) $(LIBYAML_LIBS) $(LIBCURL_LIBS) $(LIBRSYNC_LIBS) # The lib providing sd_listen_fds() is not needed in libpromises, it's actually # needed in libcfnet. But adding it here is an easy way to make sure it's