Skip to content

Commit 80dc43a

Browse files
authored
Merge branch 'main' into s3-input-change
2 parents d43c0de + ee35748 commit 80dc43a

File tree

119 files changed

+698
-736
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

119 files changed

+698
-736
lines changed

.github/workflows/clp-core-build-macos.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ on:
1313
- "components/core/tools/scripts/deps-download/**"
1414
- "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
1515
- "taskfile.yaml"
16-
- "taskfiles/deps.yaml"
16+
- "taskfiles/**"
1717
- "tools/scripts/deps-download/**"
1818
push:
1919
paths:
@@ -27,7 +27,7 @@ on:
2727
- "components/core/tools/scripts/deps-download/**"
2828
- "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
2929
- "taskfile.yaml"
30-
- "taskfiles/deps.yaml"
30+
- "taskfiles/**"
3131
- "tools/scripts/deps-download/**"
3232
schedule:
3333
# Run daily at 00:15 UTC (the 15 is to avoid periods of high load)

.github/workflows/clp-core-build.yaml

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ on:
99
- ".gitmodules"
1010
- "components/core/**"
1111
- "taskfile.yaml"
12-
- "taskfiles/deps.yaml"
12+
- "taskfiles/**"
1313
- "tools/scripts/deps-download/**"
1414
- "!components/core/tools/scripts/lib_install/macos/**"
1515
push:
@@ -20,7 +20,7 @@ on:
2020
- ".gitmodules"
2121
- "components/core/**"
2222
- "taskfile.yaml"
23-
- "taskfiles/deps.yaml"
23+
- "taskfiles/**"
2424
- "tools/scripts/deps-download/**"
2525
- "!components/core/tools/scripts/lib_install/macos/**"
2626
schedule:
@@ -32,8 +32,12 @@ env:
3232
BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-"
3333
DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-"
3434

35-
# Currency group to prevent multiple workflow instances from trying to publish container images
36-
concurrency: "${{github.workflow}}-${{github.ref}}"
35+
concurrency:
36+
group: "${{github.workflow}}-${{github.ref}}"
37+
38+
# Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted
39+
# publishing of container images.
40+
cancel-in-progress: "${{github.ref != 'refs/heads/main'}}"
3741

3842
jobs:
3943
filter-relevant-changes:
@@ -88,7 +92,7 @@ jobs:
8892
- "components/core/tests/**"
8993
- "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
9094
- "taskfile.yaml"
91-
- "taskfiles/deps.yaml"
95+
- "taskfiles/**"
9296
- "tools/scripts/deps-download/**"
9397
9498
centos-stream-9-deps-image:

.gitmodules

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,3 @@
1-
[submodule "components/core/submodules/Catch2"]
2-
path = components/core/submodules/Catch2
3-
url = https://github.com/catchorg/Catch2.git
4-
[submodule "components/core/submodules/date"]
5-
path = components/core/submodules/date
6-
url = https://github.com/HowardHinnant/date.git
7-
[submodule "components/core/submodules/json"]
8-
path = components/core/submodules/json
9-
url = https://github.com/nlohmann/json.git
10-
shallow = true
11-
[submodule "components/core/submodules/yaml-cpp"]
12-
path = components/core/submodules/yaml-cpp
13-
url = https://github.com/jbeder/yaml-cpp.git
14-
[submodule "components/core/submodules/log-surgeon"]
15-
path = components/core/submodules/log-surgeon
16-
url = https://github.com/y-scope/log-surgeon.git
17-
[submodule "components/core/submodules/simdjson"]
18-
path = components/core/submodules/simdjson
19-
url = https://github.com/simdjson/simdjson.git
20-
[submodule "components/core/submodules/abseil-cpp"]
21-
path = components/core/submodules/abseil-cpp
22-
url = https://github.com/abseil/abseil-cpp.git
231
[submodule "tools/yscope-dev-utils"]
242
path = tools/yscope-dev-utils
253
url = https://github.com/y-scope/yscope-dev-utils.git
26-
[submodule "components/core/submodules/outcome"]
27-
path = components/core/submodules/outcome
28-
url = https://github.com/ned14/outcome.git
29-
[submodule "components/core/submodules/utfcpp"]
30-
path = components/core/submodules/utfcpp
31-
url = https://github.com/nemtrif/utfcpp.git
32-
[submodule "components/core/submodules/ystdlib-cpp"]
33-
path = components/core/submodules/ystdlib-cpp
34-
url = https://github.com/y-scope/ystdlib-cpp.git

components/clp-py-utils/clp_py_utils/clp_metadata_db_utils.py

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -81,10 +81,10 @@ def _create_files_table(db_cursor, table_prefix: str) -> None:
8181
)
8282

8383

84-
def _create_column_metadata_table(db_cursor, table_name: str) -> None:
84+
def _create_column_metadata_table(db_cursor, table_prefix: str) -> None:
8585
db_cursor.execute(
8686
f"""
87-
CREATE TABLE IF NOT EXISTS `{table_name}` (
87+
CREATE TABLE IF NOT EXISTS `{table_prefix}{COLUMN_METADATA_TABLE_SUFFIX}` (
8888
`name` VARCHAR(512) NOT NULL,
8989
`type` TINYINT NOT NULL,
9090
PRIMARY KEY (`name`, `type`)
@@ -126,17 +126,13 @@ def create_metadata_db_tables(db_cursor, table_prefix: str, dataset: str | None
126126
tags_table_name = f"{table_prefix}{TAGS_TABLE_SUFFIX}"
127127
archive_tags_table_name = f"{table_prefix}{ARCHIVE_TAGS_TABLE_SUFFIX}"
128128

129-
# TODO: Update this to
130-
# {table_prefix}{CLP_DEFAULT_DATASET_NAME}_{COLUMN_METADATA_TABLE_SUFFIX} when we can also
131-
# change the indexer to match.
132-
column_metadata_table_name = (
133-
f"{table_prefix}{COLUMN_METADATA_TABLE_SUFFIX}_{CLP_DEFAULT_DATASET_NAME}"
134-
)
135-
136129
_create_archives_table(db_cursor, archives_table_name)
137130
_create_tags_table(db_cursor, tags_table_name)
138131
_create_archive_tags_table(
139132
db_cursor, archive_tags_table_name, archives_table_name, tags_table_name
140133
)
141134
_create_files_table(db_cursor, table_prefix)
142-
_create_column_metadata_table(db_cursor, column_metadata_table_name)
135+
136+
# TODO: Create this table only for the `CLP_S` storage-engine after the dataset feature is
137+
# fully implemented.
138+
_create_column_metadata_table(db_cursor, f"{table_prefix}{CLP_DEFAULT_DATASET_NAME}_")

components/core/.clang-format

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ IncludeCategories:
44
# NOTE: A header is grouped by first matching regex
55
# Library headers. Update when adding new libraries.
66
# NOTE: clang-format retains leading white-space on a line in violation of the YAML spec.
7-
- Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|curl|date|fmt|json|log_surgeon|lzma|mongocxx\
8-
|msgpack|mysql|openssl|outcome|regex_utils|simdjson|spdlog|sqlite3|string_utils|yaml-cpp|ystdlib\
9-
|zstd)"
7+
- Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|curl|date|fmt|log_surgeon|lzma|mongocxx\
8+
|msgpack|mysql|nlohmann|openssl|outcome|regex_utils|simdjson|spdlog|sqlite3|string_utils|yaml-cpp\
9+
|ystdlib|zstd)"
1010
Priority: 3
1111
# C system headers
1212
- Regex: "^<.+\\.h>"

components/core/.gitignore

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1 @@
11
build/**
2-
submodules/sqlite3/*
3-
third-party/**

components/core/CMakeLists.txt

Lines changed: 62 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,28 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
9696
endif ()
9797
endif ()
9898

99+
if(PROJECT_IS_TOP_LEVEL)
100+
# Include dependency settings if the project isn't being included as a subproject.
101+
# NOTE: We mark the file optional since it's not required if the user happens to have the
102+
# dependencies installed already.
103+
include("${CMAKE_SOURCE_DIR}/../../build/deps/core/cmake-settings/all.cmake"
104+
OPTIONAL
105+
RESULT_VARIABLE CLP_DEPS_SETTINGS_FILE_PATH
106+
)
107+
108+
if(NOT CLP_DEPS_SETTINGS_FILE_PATH STREQUAL "NOTFOUND")
109+
# Set CMP0144 since our minimum required CMake version is less than 3.27.
110+
if(POLICY "CMP0144")
111+
cmake_policy(SET "CMP0144" "NEW")
112+
endif()
113+
endif()
114+
endif()
115+
116+
find_package(absl REQUIRED)
117+
if (absl_FOUND)
118+
message(STATUS "Found absl ${absl_VERSION}")
119+
endif()
120+
99121
# Find and setup ANTLR Library
100122
# We build and link to the static library
101123
find_package(ANTLR REQUIRED)
@@ -116,6 +138,16 @@ else()
116138
message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for Boost")
117139
endif()
118140

141+
find_package(Catch2 REQUIRED)
142+
if (Catch2_FOUND)
143+
message(STATUS "Found Catch2 ${Catch2_VERSION}")
144+
endif()
145+
146+
find_package(date REQUIRED)
147+
if (date_FOUND)
148+
message(STATUS "Found date ${date_VERSION}")
149+
endif()
150+
119151
# Find and setup fmt
120152
# NOTE:
121153
# - We only try to link to the static library
@@ -127,6 +159,21 @@ else()
127159
message(FATAL_ERROR "Could not find static libraries for fmt")
128160
endif()
129161

162+
find_package(log_surgeon REQUIRED)
163+
if(log_surgeon_FOUND)
164+
message(STATUS "Found log_surgeon ${log_surgeon_VERSION}")
165+
endif()
166+
167+
find_package(nlohmann_json REQUIRED)
168+
if(nlohmann_json_FOUND)
169+
message(STATUS "Found nlohmann_json ${nlohmann_json_VERSION}")
170+
endif()
171+
172+
find_package(simdjson REQUIRED)
173+
if(simdjson_FOUND)
174+
message(STATUS "Found simdjson ${simdjson_VERSION}")
175+
endif()
176+
130177
# Find and setup spdlog
131178
if(CLP_USE_STATIC_LIBS)
132179
# NOTE: On some Linux distributions (e.g. Ubuntu), the spdlog package only contains a dynamic
@@ -175,9 +222,6 @@ else ()
175222
message(FATAL_ERROR "OpenSSL not found")
176223
endif ()
177224

178-
# Add log surgeon
179-
add_subdirectory(submodules/log-surgeon EXCLUDE_FROM_ALL)
180-
181225
# Find and setup MariaDBClient library
182226
if(CLP_USE_STATIC_LIBS)
183227
# NOTE: We can't statically link to MariaDBClient since it's GPL
@@ -212,21 +256,16 @@ else()
212256
message(FATAL_ERROR "Could not find msgpack-cxx")
213257
endif()
214258

215-
# Add abseil-cpp
216-
set(ABSL_PROPAGATE_CXX_STD ON)
217-
add_subdirectory(submodules/abseil-cpp EXCLUDE_FROM_ALL)
218-
219-
# Add simdjson
220-
add_subdirectory(submodules/simdjson EXCLUDE_FROM_ALL)
221-
222259
find_package(Threads REQUIRED)
223260

224-
# Add yaml-cpp
225-
add_subdirectory(submodules/yaml-cpp EXCLUDE_FROM_ALL)
261+
find_package(yaml-cpp REQUIRED)
262+
if(yaml-cpp_FOUND)
263+
message(STATUS "Found yaml-cpp ${yaml-cpp_VERSION}")
264+
endif()
226265

227-
# Add ystdlib-cpp
266+
# Add ystdlib
228267
set(YSTDLIB_CPP_BUILD_TESTING OFF)
229-
add_subdirectory(submodules/ystdlib-cpp EXCLUDE_FROM_ALL)
268+
add_subdirectory("${CLP_YSTDLIB_SOURCE_DIRECTORY}" "${CMAKE_BINARY_DIR}/ystdlib" EXCLUDE_FROM_ALL)
230269

231270
# Find and setup ZStd Library
232271
if(CLP_USE_STATIC_LIBS)
@@ -372,6 +411,7 @@ set(SOURCE_FILES_reducer_unitTest
372411
)
373412

374413
set(SOURCE_FILES_unitTest
414+
"${CLP_SQLITE3_SOURCE_DIRECTORY}/sqlite3.c"
375415
src/clp/aws/AwsAuthenticationSigner.cpp
376416
src/clp/aws/AwsAuthenticationSigner.hpp
377417
src/clp/aws/constants.hpp
@@ -433,6 +473,7 @@ set(SOURCE_FILES_unitTest
433473
src/clp/ffi/ir_stream/protocol_constants.hpp
434474
src/clp/ffi/ir_stream/Serializer.cpp
435475
src/clp/ffi/ir_stream/Serializer.hpp
476+
src/clp/ffi/ir_stream/search/NewProjectedSchemaTreeNodeCallbackReq.hpp
436477
src/clp/ffi/ir_stream/utils.cpp
437478
src/clp/ffi/ir_stream/utils.hpp
438479
src/clp/ffi/KeyValuePairLogEvent.cpp
@@ -594,9 +635,6 @@ set(SOURCE_FILES_unitTest
594635
src/clp/version.hpp
595636
src/clp/WriterInterface.cpp
596637
src/clp/WriterInterface.hpp
597-
submodules/sqlite3/sqlite3.c
598-
submodules/sqlite3/sqlite3.h
599-
submodules/sqlite3/sqlite3ext.h
600638
tests/LogSuppressor.hpp
601639
tests/TestOutputCleaner.hpp
602640
tests/test-BoundedReader.cpp
@@ -639,30 +677,34 @@ add_executable(unitTest
639677
)
640678
target_include_directories(unitTest
641679
PRIVATE
642-
${CMAKE_SOURCE_DIR}/submodules
680+
${CLP_OUTCOME_INCLUDE_DIRECTORY}
681+
${CLP_SQLITE3_INCLUDE_DIRECTORY}
643682
)
644683
target_link_libraries(unitTest
645684
PRIVATE
646685
absl::flat_hash_map
647686
Boost::filesystem Boost::iostreams Boost::program_options Boost::regex Boost::url
687+
Catch2::Catch2
648688
${CURL_LIBRARIES}
649689
clp_s::search::ast
650690
clp_s::TimestampPattern
691+
date::date
651692
fmt::fmt
652693
kql
653694
log_surgeon::log_surgeon
654695
LibArchive::LibArchive
655696
MariaDBClient::MariaDBClient
656697
${MONGOCXX_TARGET}
657-
simdjson
698+
nlohmann_json::nlohmann_json
699+
simdjson::simdjson
658700
spdlog::spdlog
659701
sql
660702
OpenSSL::Crypto
661703
${sqlite_LIBRARY_DEPENDENCIES}
662704
${STD_FS_LIBS}
663705
clp::regex_utils
664706
clp::string_utils
665-
yaml-cpp::yaml-cpp
707+
yaml-cpp
666708
ystdlib::containers
667709
ystdlib::error_handling
668710
${LIBLZMA_LIBRARIES}

components/core/cmake/Modules/FindANTLR.cmake

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ endif ()
1010

1111
set(ANTLR4_TAG 4.13.1)
1212
add_definitions(-DANTLR4CPP_STATIC)
13-
set(ANTLR_EXECUTABLE ${PROJECT_SOURCE_DIR}/third-party/antlr/antlr-${ANTLR4_TAG}-complete.jar)
1413
include(ExternalAntlr4Cpp)
1514

1615
find_package(Java 11 REQUIRED COMPONENTS Runtime)

components/core/src/clp/GlobalMetadataDBConfig.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#include "GlobalMetadataDBConfig.hpp"
22

33
#include <fmt/core.h>
4-
#include <yaml-cpp/include/yaml-cpp/yaml.h>
4+
#include <yaml-cpp/yaml.h>
55

66
using std::exception;
77
using std::invalid_argument;

components/core/src/clp/SQLiteDB.hpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@
33

44
#include <string>
55

6+
#include <sqlite3/sqlite3.h>
7+
68
#include "ErrorCode.hpp"
7-
#include "sqlite3/sqlite3.h"
89
#include "SQLitePreparedStatement.hpp"
910
#include "TraceableException.hpp"
1011

0 commit comments

Comments
 (0)