Merge branch 'main' into prestoPackage

Author: davemarco

components/clp-package-utils/clp_package_utils/scripts/start_clp.py

@@ -907,7 +907,6 @@ def start_webui(
         "MongoDbStreamFilesCollectionName": clp_config.results_cache.stream_collection_name,
         "ClientDir": str(container_webui_dir / "client"),
         "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"),
-        "StreamFilesDir": str(container_clp_config.stream_output.get_directory()),
         "StreamTargetUncompressedSize": container_clp_config.stream_output.target_uncompressed_size,
     }
 
@@ -916,7 +915,7 @@ def start_webui(
     stream_storage = clp_config.stream_output.storage
     if StorageType.S3 == stream_storage.type:
         s3_config = stream_storage.s3_config
-
+        server_settings_json_updates["StreamFilesDir"] = None
         server_settings_json_updates["StreamFilesS3Region"] = s3_config.region_code
         server_settings_json_updates["StreamFilesS3PathPrefix"] = (
             f"{s3_config.bucket}/{s3_config.key_prefix}"
@@ -927,6 +926,9 @@ def start_webui(
         else:
             server_settings_json_updates["StreamFilesS3Profile"] = None
     elif StorageType.FS == stream_storage.type:
+        server_settings_json_updates["StreamFilesDir"] = str(
+            container_clp_config.stream_output.get_directory()
+        )
         server_settings_json_updates["StreamFilesS3Region"] = None
         server_settings_json_updates["StreamFilesS3PathPrefix"] = None
         server_settings_json_updates["StreamFilesS3Profile"] = None
@@ -959,7 +961,6 @@ def start_webui(
     ]
     necessary_mounts = [
         mounts.clp_home,
-        mounts.stream_output_dir,
     ]
     if StorageType.S3 == stream_storage.type:
         auth = stream_storage.s3_config.aws_authentication
@@ -975,6 +976,9 @@ def start_webui(
                 necessary_mounts.append(mounts.aws_config_dir)
             if aws_env_vars:
                 necessary_env_vars.extend(aws_env_vars)
+    elif StorageType.FS == stream_storage.type:
+        necessary_mounts.append(mounts.stream_output_dir)
+
     append_docker_options(container_cmd, necessary_mounts, necessary_env_vars)
     container_cmd.append(clp_config.execution_container)
 

components/core/CMakeLists.txt

@@ -297,16 +297,14 @@ if(CLP_NEED_YSTDLIB)
     add_subdirectory("${CLP_YSTDLIB_SOURCE_DIRECTORY}" "${CMAKE_BINARY_DIR}/ystdlib" EXCLUDE_FROM_ALL)
 endif()
 
-# Find and setup ZStd Library
 if(CLP_NEED_ZSTD)
+    # v1.4.8 is the lowest version available in the package managers of the OSes we support.
+    find_package(zstd 1.4.8 REQUIRED)
+    message(STATUS "Found zstd ${zstd_VERSION}")
     if(CLP_USE_STATIC_LIBS)
-        set(ZStd_USE_STATIC_LIBS ON)
-    endif()
-    find_package(ZStd 1.4.4 REQUIRED)
-    if(ZStd_FOUND)
-        message(STATUS "Found ZStd ${ZStd_VERSION}")
+        set(zstd_TARGET zstd::libzstd_static)
     else()
-        message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for ZStd")
+        set(zstd_TARGET zstd::libzstd_shared)
     endif()
 endif()
 
@@ -402,12 +400,6 @@ set(SOURCE_FILES_clp_s_unitTest
     src/clp_s/SchemaWriter.hpp
     src/clp_s/search/AddTimestampConditions.cpp
     src/clp_s/search/AddTimestampConditions.hpp
-    src/clp_s/search/clp_search/EncodedVariableInterpreter.cpp
-    src/clp_s/search/clp_search/EncodedVariableInterpreter.hpp
-    src/clp_s/search/clp_search/Grep.cpp
-    src/clp_s/search/clp_search/Grep.hpp
-    src/clp_s/search/clp_search/Query.cpp
-    src/clp_s/search/clp_search/Query.hpp
     src/clp_s/search/EvaluateRangeIndexFilters.cpp
     src/clp_s/search/EvaluateRangeIndexFilters.hpp
     src/clp_s/search/EvaluateTimestampIndex.cpp
@@ -429,10 +421,6 @@ set(SOURCE_FILES_clp_s_unitTest
     src/clp_s/TimestampEntry.hpp
     src/clp_s/Utils.cpp
     src/clp_s/Utils.hpp
-    src/clp_s/VariableDecoder.cpp
-    src/clp_s/VariableDecoder.hpp
-    src/clp_s/VariableEncoder.cpp
-    src/clp_s/VariableEncoder.hpp
     src/clp_s/ZstdCompressor.cpp
     src/clp_s/ZstdCompressor.hpp
     src/clp_s/ZstdDecompressor.cpp
@@ -782,7 +770,7 @@ if(CLP_BUILD_TESTING)
             ystdlib::containers
             ystdlib::error_handling
             ${LIBLZMA_LIBRARIES}
-            ZStd::ZStd
+            ${zstd_TARGET}
             )
     target_compile_features(unitTest
             PRIVATE cxx_std_20

components/core/cmake/Modules/FindLZ4.cmake

@@ -249,6 +249,7 @@ function(set_clp_s_clp_dependencies_dependencies)
         CLP_NEED_BOOST
         CLP_NEED_CURL
         CLP_NEED_FMT
+        CLP_NEED_LOG_SURGEON
         CLP_NEED_MSGPACKCXX
         CLP_NEED_NLOHMANN_JSON
         CLP_NEED_OPENSSL
@@ -311,6 +312,7 @@ endfunction()
 function(set_clp_s_search_dependencies)
     set_clp_need_flags(
         CLP_NEED_ABSL
+        CLP_NEED_LOG_SURGEON
         CLP_NEED_SIMDJSON
         CLP_NEED_SPDLOG
     )

components/core/cmake/Modules/FindZStd.cmake

@@ -7,8 +7,6 @@
 #include <cassert>
 #include <cerrno>
 
-#include <boost/filesystem.hpp>
-
 using std::string;
 
 namespace clp {

components/core/cmake/Options/options.cmake

@@ -150,7 +150,7 @@ if(CLP_BUILD_EXECUTABLES)
                 yaml-cpp
                 ystdlib::containers
                 ystdlib::error_handling
-                ZStd::ZStd
+                ${zstd_TARGET}
         )
         # Put the built executable at the root of the build directory
         set_target_properties(

components/core/src/clp/StringReader.cpp

@@ -178,7 +178,7 @@ if(CLP_BUILD_EXECUTABLES)
                 clp::string_utils
                 ystdlib::containers
                 ystdlib::error_handling
-                ZStd::ZStd
+                ${zstd_TARGET}
         )
         # Put the built executable at the root of the build directory
         set_target_properties(

components/core/src/clp/clg/CMakeLists.txt

@@ -189,7 +189,7 @@ if(CLP_BUILD_EXECUTABLES)
                 yaml-cpp
                 ystdlib::containers
                 ystdlib::error_handling
-                ZStd::ZStd
+                ${zstd_TARGET}
         )
         # Put the built executable at the root of the build directory
         set_target_properties(

components/core/src/clp/clo/CMakeLists.txt

@@ -23,12 +23,6 @@ bool is_delim(signed char c) {
                || ('A' <= c && c <= 'Z') || '\\' == c || '_' == c || ('a' <= c && c <= 'z'));
 }
 
-bool is_variable_placeholder(char c) {
-    return (enum_to_underlying_type(VariablePlaceholder::Integer) == c)
-           || (enum_to_underlying_type(VariablePlaceholder::Dictionary) == c)
-           || (enum_to_underlying_type(VariablePlaceholder::Float) == c);
-}
-
 bool is_var(std::string_view value) {
     size_t begin_pos = 0;
     size_t end_pos = 0;