Data compression between the CH servers and the drivers (#523)

* support_compression: initial

* support_compression: slightly works

* support_compression: LZ4 seems to be working

* support_compression: code cleanup

* support_compression: RowBinaryWithNamesAndTypes, tiny cleanup

* support_compression: some doc, test

* support_compression: need_more_input + minor improvements

* support_compression: minor things per windsurf-bot' review

* support_compression: irrelevant session_id change reverted, minor

* support_compression: one more null pointer check

* support_compression: destruction order fix (sanitizer complain)

* support_compression: add lz4 library

* support_compression: include vector (for Win and mac)

* support_compression: tiny cleanup

* support_compression: stream owning model modified

* Revert "support_compression: stream owning model modified"

This reverts commit 8c82dc3.

* Revert "support_compression: tiny cleanup"

This reverts commit 638008d.

* Revert "support_compression: include vector (for Win and mac)"

This reverts commit 9e98da6.

* Revert "support_compression: add lz4 library"

This reverts commit 19a0829.

* Revert "support_compression: destruction order fix (sanitizer complain)"

This reverts commit b15b466.

* Revert "support_compression: one more null pointer check"

This reverts commit b31874f.

* Revert "support_compression: irrelevant session_id change reverted, minor"

This reverts commit db61e10.

* Revert "support_compression: minor things per windsurf-bot' review"

This reverts commit 937b78f.

* Revert "support_compression: need_more_input + minor improvements"

This reverts commit 347db71.

* Revert "support_compression: RowBinaryWithNamesAndTypes, tiny cleanup"

This reverts commit a089e5d.

* Revert "support_compression: code cleanup"

This reverts commit da17f18.

* Revert "support_compression: LZ4 seems to be working"

This reverts commit 7b92ea0.

* support_compression: reapplied everything non-LZ4 related

Author: ilejn

README.md

@@ -109,8 +109,9 @@ The list of DSN parameters recognized by the driver is as follows:
 |    `HugeIntAsString`    |                                                          `off`                                                           | Report integer column types that may underflow or overflow 64-bit signed integer (`SQL_BIGINT`) as a `String`/`SQL_VARCHAR`                                                                                                                                                                                                                                                                                                  |
 |       `DriverLog`       |                                  `on` if `CMAKE_BUILD_TYPE` is `Debug`, `off` otherwise                                  | Enable or disable the extended driver logging                                                                                                                                                                                                                                                                                                                                                                                |
 |     `DriverLogFile`     |               `\temp\clickhouse-odbc-driver.log`  on Windows, `/tmp/clickhouse-odbc-driver.log` otherwise                | Path to the extended driver log file (used when `DriverLog` is `on`)                                                                                                                                                                                                                                                                                                                                                         |
-|     `AutoSessionId`     |                                                          `off`                                                           | Auto generate session_id required to use some features of CH (e.g. TEMPORARY TABLE)                                                                            |
-|      `ClientName`       |                                                          empty                                                           | Sets additional information about the calling application. This string will be used as a prefix for the User-Agent header.
+|     `AutoSessionId`     |                                                          `off`                                                           | Auto generate session_id required to use some features of CH (e.g. TEMPORARY TABLE)                                                                                                                                                                                                                                                                                                                                          |
+|      `ClientName`       |                                                          empty                                                           | Sets additional information about the calling application. This string will be used as a prefix for the User-Agent header.                                                                                                                                                                                                                                                                                                   |
+|       `Compress`        |                                                          `off`                                                           | Pass enable_http_compression=1 parameter to server                                                                                                                                                                                                                                                                                                                                                                           |
 
 
 ### URL query string

driver/config/config.cpp

@@ -253,7 +253,9 @@ key_value_map_t readDSNInfo(const std::string & dsn_utf8) {
             INI_DRIVERLOG,
             INI_DRIVERLOGFILE,
             INI_AUTO_SESSION_ID,
-            INI_CLIENT_NAME
+            INI_CLIENT_NAME,
+            INI_COMPRESS,
+            INI_USE_COMPRESSION
         }
     ) {
         if (

driver/config/ini_defines.h

@@ -33,6 +33,8 @@
 #define INI_DRIVERLOGFILE   "DriverLogFile"
 #define INI_AUTO_SESSION_ID "AutoSessionId"
 #define INI_CLIENT_NAME     "ClientName"
+#define INI_COMPRESS        "Compress"
+#define INI_USE_COMPRESSION "UseCompression"
 
 #if defined(UNICODE)
 #   define INI_DSN_DEFAULT          DSN_DEFAULT_UNICODE
@@ -54,6 +56,8 @@
 #define INI_STRINGMAXLENGTH_DEFAULT "1048575"
 #define INI_AUTO_SESSION_ID_DEFAULT "off"
 #define INI_CLIENT_NAME_DEFAULT     ""
+#define INI_COMPRESS_DEFAULT        "0"
+#define INI_USE_COMPRESSION_DEFAULT "0"
 
 #ifdef NDEBUG
 #    define INI_DRIVERLOG_DEFAULT "off"

driver/connection.cpp

@@ -93,6 +93,7 @@ Poco::URI Connection::getUri() const {
     bool database_set = false;
     bool default_format_set = false;
     bool session_id_set = false;
+    bool enable_http_compression_set = false;
 
     for (const auto& parameter : uri.getQueryParameters()) {
         if (Poco::UTF8::icompare(parameter.first, "default_format") == 0) {
@@ -118,6 +119,10 @@ Poco::URI Connection::getUri() const {
         uri.addQueryParameter("session_id", session_id);
     }
 
+    if (enable_http_compression && !enable_http_compression_set) {
+        uri.addQueryParameter("enable_http_compression", enable_http_compression ? "1" : "0");
+    }
+
     return uri;
 }
 
@@ -409,6 +414,20 @@ void Connection::setConfiguration(const key_value_map_t & cs_fields, const key_v
                 client_name = value;
             }
         }
+        else if (Poco::UTF8::icompare(key, INI_COMPRESS) == 0 || Poco::UTF8::icompare(key, INI_USE_COMPRESSION) == 0) {
+            recognized_key = true;
+            unsigned int typed_value = 0;
+            valid_value =
+                (value.empty() ||
+                (
+                    Poco::NumberParser::tryParseUnsigned(value, typed_value) &&
+                    (typed_value == 1 || typed_value == 0)
+                 ) ||
+                isYesOrNo(value));
+            if (valid_value) {
+                enable_http_compression = (typed_value == 1 || isYes(value));
+            }
+        }
 
         return std::make_tuple(recognized_key, valid_value);
     };
@@ -426,6 +445,8 @@ void Connection::setConfiguration(const key_value_map_t & cs_fields, const key_v
             const auto & recognized_key = std::get<0>(res);
             const auto & valid_value = std::get<1>(res);
 
+            // LOG("DSN: known attribute '" << key << "', valid value, '" << valid_value << "'");
+
             if (recognized_key) {
                 if (!valid_value)
                     throw std::runtime_error("DSN: bad value '" + value + "' for attribute '" + key + "'");

driver/connection.h

@@ -43,6 +43,7 @@ class Connection
     std::int32_t stringmaxlength = 0;
     bool auto_session_id = false;
     std::string client_name;
+    bool enable_http_compression = false;
 
 public:
     std::unique_ptr<Poco::Net::HTTPClientSession> session;

driver/format/ODBCDriver2.cpp

@@ -263,6 +263,15 @@ ODBCDriver2ResultReader::ODBCDriver2ResultReader(
     result_set = std::make_unique<ODBCDriver2ResultSet>(timezone, stream, releaseMutator());
 }
 
+ODBCDriver2ResultReader::ODBCDriver2ResultReader(const std::string & timezone_, std::istream & raw_stream, std::unique_ptr<ResultMutator> && mutator, std::unique_ptr<std::istream> && inflating_input_stream)
+  : ResultReader(timezone_, raw_stream, std::move(mutator), std::move(inflating_input_stream))
+{
+    if (stream.eof())
+        return;
+
+    result_set = std::make_unique<ODBCDriver2ResultSet>(timezone, stream, releaseMutator());
+}
+
 bool ODBCDriver2ResultReader::advanceToNextResultSet() {
     // ODBCDriver2 format doesn't support multiple result sets in the response,
     // so only a basic cleanup is done here.

driver/format/ODBCDriver2.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include "driver/platform/platform.h"
 #include "driver/result_set.h"
 
 #include <Poco/Net/HTTPClientSession.h>

driver/format/RowBinaryWithNamesAndTypes.cpp

@@ -319,6 +319,16 @@ RowBinaryWithNamesAndTypesResultReader::RowBinaryWithNamesAndTypesResultReader(
     result_set = std::make_unique<RowBinaryWithNamesAndTypesResultSet>(timezone, stream, releaseMutator());
 }
 
+RowBinaryWithNamesAndTypesResultReader::RowBinaryWithNamesAndTypesResultReader(const std::string & timezone_, std::istream & raw_stream, std::unique_ptr<ResultMutator> && mutator, std::unique_ptr<std::istream> && inflating_input_stream)
+  : ResultReader(timezone_, raw_stream, std::move(mutator), std::move(inflating_input_stream))
+{
+    if (stream.eof())
+        return;
+
+    result_set = std::make_unique<RowBinaryWithNamesAndTypesResultSet>(timezone, stream, releaseMutator());
+
+}
+
 bool RowBinaryWithNamesAndTypesResultReader::advanceToNextResultSet() {
     // RowBinaryWithNamesAndTypes format doesn't support multiple result sets in the response,
     // so only a basic cleanup is done here.

driver/result_set.cpp

@@ -1,4 +1,5 @@
 #include "driver/result_set.h"
+#include "driver/driver.h"
 #include "driver/format/ODBCDriver2.h"
 #include "driver/format/RowBinaryWithNamesAndTypes.h"
 
@@ -313,6 +314,13 @@ ResultReader::ResultReader(
 {
 }
 
+ResultReader::ResultReader(const std::string & timezone_, std::istream & raw_stream, std::unique_ptr<ResultMutator> && mutator, std::unique_ptr<std::istream> && inflating_input_stream_)
+    : timezone(timezone_)
+    , stream(raw_stream, std::move(inflating_input_stream_))
+    , result_mutator(std::move(mutator))
+{
+}
+
 bool ResultReader::hasResultSet() const {
     return static_cast<bool>(result_set);
 }

driver/result_set.h

@@ -169,7 +169,8 @@ std::unique_ptr<ResultReader> make_result_reader(
 );
 
 template <typename ConversionContext>
-SQLRETURN Field::extract(BindingInfo & binding_info, ConversionContext && context) const {
+SQLRETURN Field::extract(BindingInfo & binding_info, ConversionContext && context) const
+{
     return std::visit([&binding_info, &context] (auto & value) {
         if constexpr (std::is_same_v<DataSourceType<DataSourceTypeId::Nothing>, std::decay_t<decltype(value)>>) {
             return fillOutputNULL(binding_info.value, binding_info.value_max_size, binding_info.indicator);