Skip to content

refactor(core): Use C++20 concepts to enforce requirements for logtype and variable dictionary entries, readers, and writers. #1267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Sep 17, 2025
Merged

refactor(core): Use C++20 concepts to enforce requirements for logtype and variable dictionary entries, readers, and writers. #1267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
9404cb4
Add concepts for the variable and logtype dictionaries and their entries
gibber9809 Jul 29, 2025
6ed6d74
Use dictionary concepts in EncodedVariableInterpreter.hpp
gibber9809 Aug 26, 2025
e9dbd71
Use dictionary concepts in GrepCore.hpp
gibber9809 Aug 26, 2025
0131b2d
Address rabbit comments.
gibber9809 Aug 26, 2025
7da8133
Fix instance of clp-s dictionary readers not following dictionary con…
gibber9809 Aug 26, 2025
2d60d5f
Merge branch 'main' into clp-dictionary-concepts
gibber9809 Aug 28, 2025
d8d81f8
Apply suggestions from code review
gibber9809 Sep 15, 2025
25d7a2f
Merge remote-tracking branch 'upstream/main' into clp-dictionary-conc…
gibber9809 Sep 15, 2025
454daba
Apply some review comments.
gibber9809 Sep 15, 2025
eb6e746
Simplify some templates
gibber9809 Sep 15, 2025
0da5e24
Remove unused parameter in concept
gibber9809 Sep 15, 2025
0d1e320
Rename entry_t to EntryT
gibber9809 Sep 15, 2025
29226f1
Rename EntryT -> Entry
gibber9809 Sep 15, 2025
80e27c6
Rename placeholder_ix -> placeholder_idx
gibber9809 Sep 15, 2025
219daed
Merge remote-tracking branch 'upstream/main' into clp-dictionary-conc…
gibber9809 Sep 15, 2025
f9cb84c
Split new dictionary concepts into dedicated header files.
gibber9809 Sep 17, 2025
abc5387
Merge remote-tracking branch 'upstream/main' into clp-dictionary-conc…
gibber9809 Sep 17, 2025
3201f95
Address rabbit comment
gibber9809 Sep 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions components/core/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -580,7 +580,9 @@ set(SOURCE_FILES_unitTest
src/clp/LogSurgeonReader.hpp
src/clp/LogTypeDictionaryEntry.cpp
src/clp/LogTypeDictionaryEntry.hpp
src/clp/LogTypeDictionaryEntryReq.hpp
src/clp/LogTypeDictionaryReader.hpp
src/clp/LogTypeDictionaryReaderReq.hpp
src/clp/LogTypeDictionaryWriter.cpp
src/clp/LogTypeDictionaryWriter.hpp
src/clp/math_utils.hpp
Expand Down Expand Up @@ -673,9 +675,12 @@ set(SOURCE_FILES_unitTest
src/clp/Utils.hpp
src/clp/VariableDictionaryEntry.cpp
src/clp/VariableDictionaryEntry.hpp
src/clp/VariableDictionaryEntryReq.hpp
src/clp/VariableDictionaryReader.hpp
src/clp/VariableDictionaryReaderReq.hpp
src/clp/VariableDictionaryWriter.cpp
src/clp/VariableDictionaryWriter.hpp
src/clp/VariableDictionaryWriterReq.hpp
src/clp/version.hpp
src/clp/WriterInterface.cpp
src/clp/WriterInterface.hpp
Expand Down
2 changes: 1 addition & 1 deletion components/core/src/clp/DictionaryReader.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class DictionaryReader {
};

using dictionary_id_t = DictionaryIdType;
using entry_t = EntryType;
using Entry = EntryType;

// Constructors
DictionaryReader() : m_is_open(false), m_num_segments_read_from_index(0) {
Expand Down
2 changes: 1 addition & 1 deletion components/core/src/clp/DictionaryWriter.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ class DictionaryWriter {
};

using dictionary_id_t = DictionaryIdType;
using entry_t = EntryType;
using Entry = EntryType;

// Constructors
DictionaryWriter() : m_is_open(false) {}
Expand Down
56 changes: 34 additions & 22 deletions components/core/src/clp/EncodedVariableInterpreter.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,13 @@
#include "ffi/ir_stream/decoding_methods.hpp"
#include "ir/LogEvent.hpp"
#include "ir/types.hpp"
#include "LogTypeDictionaryEntryReq.hpp"
#include "Query.hpp"
#include "spdlog_with_specializations.hpp"
#include "TraceableException.hpp"
#include "type_utils.hpp"
#include "VariableDictionaryReaderReq.hpp"
#include "VariableDictionaryWriterReq.hpp"

namespace clp {
/**
Expand Down Expand Up @@ -123,7 +126,9 @@ class EncodedVariableInterpreter {
* @param encoded_vars
* @param var_ids
*/
template <typename LogTypeDictionaryEntryType, typename VariableDictionaryWriterType>
template <
LogTypeDictionaryEntryReq LogTypeDictionaryEntryType,
VariableDictionaryWriterReq VariableDictionaryWriterType>
static void encode_and_add_to_dictionary(
std::string_view message,
LogTypeDictionaryEntryType& logtype_dict_entry,
Expand All @@ -149,8 +154,8 @@ class EncodedVariableInterpreter {
*/
template <
typename EncodedVariableType,
typename LogTypeDictionaryEntryType,
typename VariableDictionaryWriterType>
LogTypeDictionaryEntryReq LogTypeDictionaryEntryType,
VariableDictionaryWriterReq VariableDictionaryWriterType>
static void encode_and_add_to_dictionary(
ir::LogEvent<EncodedVariableType> const& log_event,
LogTypeDictionaryEntryType& logtype_dict_entry,
Expand All @@ -172,8 +177,8 @@ class EncodedVariableInterpreter {
* @return true if successful, false otherwise
*/
template <
typename LogTypeDictionaryEntryType,
typename VariableDictionaryReaderType,
LogTypeDictionaryEntryReq LogTypeDictionaryEntryType,
VariableDictionaryReaderReq VariableDictionaryReaderType,
typename EncodedVariableContainerType>
static bool decode_variables_into_message(
LogTypeDictionaryEntryType const& logtype_dict_entry,
Expand All @@ -195,7 +200,7 @@ class EncodedVariableInterpreter {
* dictionary
* @return false otherwise
*/
template <typename VariableDictionaryReaderType>
template <VariableDictionaryReaderReq VariableDictionaryReaderType>
static bool encode_and_search_dictionary(
std::string_view var_str,
VariableDictionaryReaderType const& var_dict,
Expand All @@ -207,16 +212,13 @@ class EncodedVariableInterpreter {
* Search for the given string-form variable in the variable dictionary, encode any matches, and
* add them to the given sub-query.
* @tparam VariableDictionaryReaderType
* @tparam VariableDictionaryEntryType
* @param var_wildcard_str
* @param var_dict
* @param ignore_case
* @param sub_query
* @return true if any match found, false otherwise
*/
template <
typename VariableDictionaryReaderType,
typename VariableDictionaryEntryType = typename VariableDictionaryReaderType::entry_t>
template <VariableDictionaryReaderReq VariableDictionaryReaderType>
static bool wildcard_search_dictionary_and_get_encoded_matches(
std::string_view var_wildcard_str,
VariableDictionaryReaderType const& var_dict,
Expand All @@ -237,7 +239,9 @@ class EncodedVariableInterpreter {
* variable)
* @return The encoded variable
*/
template <typename LogTypeDictionaryEntryType, typename VariableDictionaryWriterType>
template <
LogTypeDictionaryEntryReq LogTypeDictionaryEntryType,
VariableDictionaryWriterReq VariableDictionaryWriterType>
static encoded_variable_t encode_var(
std::string_view var,
LogTypeDictionaryEntryType& logtype_dict_entry,
Expand All @@ -256,7 +260,9 @@ class EncodedVariableInterpreter {
* @param var_ids A container to add the dictionary ID to
* @return The dictionary ID
*/
template <typename LogTypeDictionaryEntryType, typename VariableDictionaryWriterType>
template <
LogTypeDictionaryEntryReq LogTypeDictionaryEntryType,
VariableDictionaryWriterReq VariableDictionaryWriterType>
static variable_dictionary_id_t add_dict_var(
std::string_view var,
LogTypeDictionaryEntryType& logtype_dict_entry,
Expand All @@ -265,7 +271,9 @@ class EncodedVariableInterpreter {
);
};

template <typename LogTypeDictionaryEntryType, typename VariableDictionaryWriterType>
template <
LogTypeDictionaryEntryReq LogTypeDictionaryEntryType,
VariableDictionaryWriterReq VariableDictionaryWriterType>
void EncodedVariableInterpreter::encode_and_add_to_dictionary(
std::string_view message,
LogTypeDictionaryEntryType& logtype_dict_entry,
Expand All @@ -289,8 +297,8 @@ void EncodedVariableInterpreter::encode_and_add_to_dictionary(

template <
typename EncodedVariableType,
typename LogTypeDictionaryEntryType,
typename VariableDictionaryWriterType>
LogTypeDictionaryEntryReq LogTypeDictionaryEntryType,
VariableDictionaryWriterReq VariableDictionaryWriterType>
void EncodedVariableInterpreter::encode_and_add_to_dictionary(
ir::LogEvent<EncodedVariableType> const& log_event,
LogTypeDictionaryEntryType& logtype_dict_entry,
Expand Down Expand Up @@ -362,8 +370,8 @@ void EncodedVariableInterpreter::encode_and_add_to_dictionary(
}

template <
typename LogTypeDictionaryEntryType,
typename VariableDictionaryReaderType,
LogTypeDictionaryEntryReq LogTypeDictionaryEntryType,
VariableDictionaryReaderReq VariableDictionaryReaderType,
typename EncodedVariableContainerType>
bool EncodedVariableInterpreter::decode_variables_into_message(
LogTypeDictionaryEntryType const& logtype_dict_entry,
Expand Down Expand Up @@ -436,7 +444,7 @@ bool EncodedVariableInterpreter::decode_variables_into_message(
return true;
}

template <typename VariableDictionaryReaderType>
template <VariableDictionaryReaderReq VariableDictionaryReaderType>
bool EncodedVariableInterpreter::encode_and_search_dictionary(
std::string_view var_str,
VariableDictionaryReaderType const& var_dict,
Expand Down Expand Up @@ -485,15 +493,15 @@ bool EncodedVariableInterpreter::encode_and_search_dictionary(
return true;
}

template <typename VariableDictionaryReaderType, typename VariableDictionaryEntryType>
template <VariableDictionaryReaderReq VariableDictionaryReaderType>
bool EncodedVariableInterpreter::wildcard_search_dictionary_and_get_encoded_matches(
std::string_view var_wildcard_str,
VariableDictionaryReaderType const& var_dict,
bool ignore_case,
SubQuery& sub_query
) {
// Find matches
Comment on lines +496 to 503
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🧹 Nitpick (assertive)

Reserve capacity for result sets to cut rehashing in hot path.

Apply this diff:

-    // Encode matches
-    std::unordered_set<encoded_variable_t> encoded_vars;
-    std::unordered_set<variable_dictionary_id_t> var_dict_ids;
+    // Encode matches
+    std::unordered_set<encoded_variable_t> encoded_vars;
+    std::unordered_set<variable_dictionary_id_t> var_dict_ids;
+    encoded_vars.reserve(var_dict_entries.size());
+    var_dict_ids.reserve(var_dict_entries.size());

Committable suggestion skipped: line range outside the PR's diff.

std::unordered_set<VariableDictionaryEntryType const*> var_dict_entries;
std::unordered_set<typename VariableDictionaryReaderType::Entry const*> var_dict_entries;
var_dict.get_entries_matching_wildcard_string(var_wildcard_str, ignore_case, var_dict_entries);
if (var_dict_entries.empty()) {
// Not in dictionary
Expand All @@ -513,7 +521,9 @@ bool EncodedVariableInterpreter::wildcard_search_dictionary_and_get_encoded_matc
return true;
}

template <typename LogTypeDictionaryEntryType, typename VariableDictionaryWriterType>
template <
LogTypeDictionaryEntryReq LogTypeDictionaryEntryType,
VariableDictionaryWriterReq VariableDictionaryWriterType>
encoded_variable_t EncodedVariableInterpreter::encode_var(
std::string_view var,
LogTypeDictionaryEntryType& logtype_dict_entry,
Expand All @@ -532,7 +542,9 @@ encoded_variable_t EncodedVariableInterpreter::encode_var(
return encoded_var;
}

template <typename LogTypeDictionaryEntryType, typename VariableDictionaryWriterType>
template <
LogTypeDictionaryEntryReq LogTypeDictionaryEntryType,
VariableDictionaryWriterReq VariableDictionaryWriterType>
variable_dictionary_id_t EncodedVariableInterpreter::add_dict_var(
std::string_view var,
LogTypeDictionaryEntryType& logtype_dict_entry,
Expand Down
27 changes: 15 additions & 12 deletions components/core/src/clp/GrepCore.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@
#include "EncodedVariableInterpreter.hpp"
#include "ir/parsing.hpp"
#include "ir/types.hpp"
#include "LogTypeDictionaryReaderReq.hpp"
#include "Query.hpp"
#include "QueryToken.hpp"
#include "VariableDictionaryReaderReq.hpp"

namespace clp {
class GrepCore {
Expand All @@ -42,7 +44,9 @@ class GrepCore {
* @param use_heuristic
* @return Query if it may match a message, std::nullopt otherwise
*/
template <typename LogTypeDictionaryReaderType, typename VariableDictionaryReaderType>
template <
LogTypeDictionaryReaderReq LogTypeDictionaryReaderType,
VariableDictionaryReaderReq VariableDictionaryReaderType>
static std::optional<Query> process_raw_query(
LogTypeDictionaryReaderType const& logtype_dict,
VariableDictionaryReaderType const& var_dict,
Expand Down Expand Up @@ -107,7 +111,7 @@ class GrepCore {
* @param logtype
* @return true if this token might match a message, false otherwise
*/
template <typename VariableDictionaryReaderType>
template <VariableDictionaryReaderReq VariableDictionaryReaderType>
static bool process_var_token(
QueryToken const& query_token,
VariableDictionaryReaderType const& var_dict,
Expand All @@ -120,7 +124,6 @@ class GrepCore {
* Generates logtypes and variables for subquery.
* @tparam LogTypeDictionaryReaderType
* @tparam VariableDictionaryReaderType
* @tparam LogTypeDictionaryEntryType
* @param logtype_dict
* @param var_dict
* @param processed_search_string
Expand All @@ -132,9 +135,8 @@ class GrepCore {
* @return SubQueryMatchabilityResult::MayMatch
*/
template <
typename LogTypeDictionaryReaderType,
typename VariableDictionaryReaderType,
typename LogTypeDictionaryEntryType = typename LogTypeDictionaryReaderType::entry_t>
LogTypeDictionaryReaderReq LogTypeDictionaryReaderType,
VariableDictionaryReaderReq VariableDictionaryReaderType>
static SubQueryMatchabilityResult generate_logtypes_and_vars_for_subquery(
LogTypeDictionaryReaderType const& logtype_dict,
VariableDictionaryReaderType const& var_dict,
Expand All @@ -145,7 +147,9 @@ class GrepCore {
);
};

template <typename LogTypeDictionaryReaderType, typename VariableDictionaryReaderType>
template <
LogTypeDictionaryReaderReq LogTypeDictionaryReaderType,
VariableDictionaryReaderReq VariableDictionaryReaderType>
std::optional<Query> GrepCore::process_raw_query(
LogTypeDictionaryReaderType const& logtype_dict,
VariableDictionaryReaderType const& var_dict,
Expand Down Expand Up @@ -266,7 +270,7 @@ std::optional<Query> GrepCore::process_raw_query(
};
}

template <typename VariableDictionaryReaderType>
template <VariableDictionaryReaderReq VariableDictionaryReaderType>
bool GrepCore::process_var_token(
QueryToken const& query_token,
VariableDictionaryReaderType const& var_dict,
Expand Down Expand Up @@ -328,9 +332,8 @@ bool GrepCore::process_var_token(
}

template <
typename LogTypeDictionaryReaderType,
typename VariableDictionaryReaderType,
typename LogTypeDictionaryEntryType>
LogTypeDictionaryReaderReq LogTypeDictionaryReaderType,
VariableDictionaryReaderReq VariableDictionaryReaderType>
GrepCore::SubQueryMatchabilityResult GrepCore::generate_logtypes_and_vars_for_subquery(
LogTypeDictionaryReaderType const& logtype_dict,
VariableDictionaryReaderType const& var_dict,
Expand Down Expand Up @@ -407,7 +410,7 @@ GrepCore::SubQueryMatchabilityResult GrepCore::generate_logtypes_and_vars_for_su
}

// Find matching logtypes
std::unordered_set<LogTypeDictionaryEntryType const*> possible_logtype_entries;
std::unordered_set<typename LogTypeDictionaryReaderType::Entry const*> possible_logtype_entries;
logtype_dict
.get_entries_matching_wildcard_string(logtype, ignore_case, possible_logtype_entries);
if (possible_logtype_entries.empty()) {
Expand Down
Loading
Loading