Add dynamic programming approach for schema search.

components/core/CMakeLists.txt

@@ -424,6 +424,8 @@ set(SOURCE_FILES_unitTest
         src/clp/Profiler.hpp
         src/clp/Query.cpp
         src/clp/Query.hpp
+        src/clp/QueryInterpretation.cpp
+        src/clp/QueryInterpretation.hpp
         src/clp/ReaderInterface.cpp
         src/clp/ReaderInterface.hpp
         src/clp/ReadOnlyMemoryMappedFile.cpp
@@ -489,6 +491,8 @@ set(SOURCE_FILES_unitTest
         src/clp/VariableDictionaryWriter.cpp
         src/clp/VariableDictionaryWriter.hpp
         src/clp/version.hpp
+        src/clp/WildcardExpression.cpp
+        src/clp/WildcardExpression.hpp
         src/clp/WriterInterface.cpp
         src/clp/WriterInterface.hpp
         submodules/sqlite3/sqlite3.c

components/core/src/clp/Grep.hpp

@@ -8,10 +8,13 @@
 
 #include "Defs.h"
 #include "Query.hpp"
+#include "QueryInterpretation.hpp"
 #include "streaming_archive/reader/Archive.hpp"
 #include "streaming_archive/reader/File.hpp"
+#include "WildcardExpression.hpp"
 
 namespace clp {
+
 class Grep {
 public:
     // Types
@@ -37,8 +40,7 @@ class Grep {
      * @param search_begin_ts
      * @param search_end_ts
      * @param ignore_case
-     * @param forward_lexer DFA for determining if input is in the schema
-     * @param reverse_lexer DFA for determining if reverse of input is in the schema
+     * @param lexer DFA for determining if input is in the schema
      * @param use_heuristic
      * @return Query if it may match a message, std::nullopt otherwise
      */
@@ -48,8 +50,7 @@ class Grep {
             epochtime_t search_begin_ts,
             epochtime_t search_end_ts,
             bool ignore_case,
-            log_surgeon::lexers::ByteLexer& forward_lexer,
-            log_surgeon::lexers::ByteLexer& reverse_lexer,
+            log_surgeon::lexers::ByteLexer& lexer,
             bool use_heuristic
     );
 
@@ -69,25 +70,6 @@ class Grep {
             bool& is_var
     );
 
-    /**
-     * Returns bounds of next potential variable (either a definite variable or a token with
-     * wildcards)
-     * @param value String containing token
-     * @param begin_pos Begin position of last token, changes to begin position of next token
-     * @param end_pos End position of last token, changes to end position of next token
-     * @param is_var Whether the token is definitely a variable
-     * @param forward_lexer DFA for determining if input is in the schema
-     * @param reverse_lexer DFA for determining if reverse of input is in the schema
-     * @return true if another potential variable was found, false otherwise
-     */
-    static bool get_bounds_of_next_potential_var(
-            std::string const& value,
-            size_t& begin_pos,
-            size_t& end_pos,
-            bool& is_var,
-            log_surgeon::lexers::ByteLexer& forward_lexer,
-            log_surgeon::lexers::ByteLexer& reverse_lexer
-    );
     /**
      * Marks which sub-queries in each query are relevant to the given file
      * @param compressed_file
@@ -126,6 +108,7 @@ class Grep {
             streaming_archive::reader::Message& compressed_msg,
             std::string& decompressed_msg
     );
+
     /**
      * Searches a file with the given query without outputting the results
      * @param query
@@ -143,6 +126,63 @@ class Grep {
             streaming_archive::reader::Archive& archive,
             streaming_archive::reader::File& compressed_file
     );
+
+    /**
+     * Generates all possible logtypes that can match each substr(0,n) of the search string.
+     * Requires that processed_search_string is valid, meaning that only wildcards are escaped
+     * and the string does not end with an escape character.
+     * @param processed_search_string
+     * @param lexer
+     * @return a vector of all QueryInterpretations that can match the query in
+     * processed_search_string.
+     */
+    static std::set<QueryInterpretation> generate_query_substring_interpretations(
+            WildcardExpression const& processed_search_string,
+            log_surgeon::lexers::ByteLexer& lexer
+    );
+
+    /**
+     * Computes the tokens (static text or different types of variables) that the given wildcard
+     * expression (as a whole) could be interpreted as, generates a `QueryInterpretation` for each
+     * one, and returns the `QueryInterpretation`s.
+     * @param wildcard_expr
+     * @param lexer
+     * @return The `QueryInterpretation`s.
+     */
+    static std::vector<QueryInterpretation> get_interpretations_for_whole_wildcard_expr(
+            WildcardExpressionView const& wildcard_expr,
+            log_surgeon::lexers::ByteLexer& lexer
+    );
+
+    /**
+     * Gets the variable types that the given wildcard expression could match.
+     * @param wildcard_expr
+     * @param lexer
+     * @return A tuple:
+     * - The set of variable types that the wildcard expression could match.
+     * - Whether the wildcard expression contains a wildcard.
+     */
+    static std::tuple<std::set<uint32_t>, bool> get_matching_variable_types(
+            WildcardExpressionView const& wildcard_expr,
+            log_surgeon::lexers::ByteLexer const& lexer
+    );
+
+    /**
+     * Compare all possible query logtypes against the archive to determine all possible sub queries
+     * that can match against messages in the archive.
+     * @param query_interpretations
+     * @param archive
+     * @param lexer
+     * @param ignore_case
+     * @param sub_queries
+     */
+    static void generate_sub_queries(
+            std::set<QueryInterpretation> const& query_interpretations,
+            streaming_archive::reader::Archive const& archive,
+            log_surgeon::lexers::ByteLexer& lexer,
+            bool ignore_case,
+            std::vector<SubQuery>& sub_queries
+    );
 };
 }  // namespace clp
 

components/core/src/clp/Query.hpp

@@ -135,6 +135,7 @@ class SubQuery {
      * @return true if matched, false otherwise
      */
     bool matches_logtype(logtype_dictionary_id_t logtype) const;
+
     /**
      * Whether the given variables contain the subquery's variables in order (but not necessarily
      * contiguously)

components/core/src/clp/QueryInterpretation.cpp

@@ -0,0 +1,195 @@
+#include "QueryInterpretation.hpp"
+
+#include <algorithm>
+#include <cstdint>
+#include <ostream>
+#include <string>
+#include <utility>
+#include <variant>
+
+#include "Defs.h"
+#include "EncodedVariableInterpreter.hpp"
+#include "log_surgeon/Lexer.hpp"
+#include "LogTypeDictionaryEntry.hpp"
+#include "string_utils/string_utils.hpp"
+
+using log_surgeon::lexers::ByteLexer;
+using std::string;
+
+namespace clp {
+auto VariableQueryToken::operator<(VariableQueryToken const& rhs) const -> bool {
+    if (m_variable_type < rhs.m_variable_type) {
+        return true;
+    }
+    if (m_variable_type > rhs.m_variable_type) {
+        return false;
+    }
+    if (m_query_substring < rhs.m_query_substring) {
+        return true;
+    }
+    if (m_query_substring > rhs.m_query_substring) {
+        return false;
+    }
+    if (m_has_wildcard != rhs.m_has_wildcard) {
+        return rhs.m_has_wildcard;
+    }
+    if (m_is_encoded != rhs.m_is_encoded) {
+        return rhs.m_is_encoded;
+    }
+    return false;
+}
+
+auto VariableQueryToken::operator>(VariableQueryToken const& rhs) const -> bool {
+    if (m_variable_type > rhs.m_variable_type) {
+        return true;
+    }
+    if (m_variable_type < rhs.m_variable_type) {
+        return false;
+    }
+    if (m_query_substring > rhs.m_query_substring) {
+        return true;
+    }
+    if (m_query_substring < rhs.m_query_substring) {
+        return false;
+    }
+    if (m_has_wildcard != rhs.m_has_wildcard) {
+        return m_has_wildcard;
+    }
+    if (m_is_encoded != rhs.m_is_encoded) {
+        return m_is_encoded;
+    }
+    return false;
+}
+
+void QueryInterpretation::append_logtype(QueryInterpretation& suffix) {
+    auto const& first_new_token = suffix.m_logtype[0];
+    if (auto& prev_token = m_logtype.back();
+        false == m_logtype.empty() && std::holds_alternative<StaticQueryToken>(prev_token)
+        && false == suffix.m_logtype.empty()
+        && std::holds_alternative<StaticQueryToken>(first_new_token))
+    {
+        std::get<StaticQueryToken>(prev_token).append(std::get<StaticQueryToken>(first_new_token));
+        m_logtype.insert(m_logtype.end(), suffix.m_logtype.begin() + 1, suffix.m_logtype.end());
+    } else {
+        m_logtype.insert(m_logtype.end(), suffix.m_logtype.begin(), suffix.m_logtype.end());
+    }
+}
+
+void QueryInterpretation::generate_logtype_string(ByteLexer& lexer) {
+    // Convert each query logtype into a set of logtype strings. Logtype strings are used in the
+    // sub query as they have the correct format for comparing against the archive. Also, a
+    // single query logtype might represent multiple logtype strings. While static text converts
+    // one-to-one, wildcard variables that may be encoded have different logtype strings when
+    // comparing against the dictionary than they do when comparing against the segment.
+
+    // Reserve size for m_logtype_string
+    uint32_t logtype_string_size = 0;
+    for (uint32_t i = 0; i < get_logtype_size(); i++) {
+        if (auto const& logtype_token = get_logtype_token(i);
+            std::holds_alternative<StaticQueryToken>(logtype_token))
+        {
+            logtype_string_size
+                    += std::get<StaticQueryToken>(logtype_token).get_query_substring().size();
+        } else {
+            logtype_string_size++;
+        }
+    }
+    m_logtype_string.reserve(logtype_string_size);
+
+    for (uint32_t i = 0; i < get_logtype_size(); i++) {
+        if (auto const& logtype_token = get_logtype_token(i);
+            std::holds_alternative<StaticQueryToken>(logtype_token))
+        {
+            m_logtype_string += std::get<StaticQueryToken>(logtype_token).get_query_substring();
+        } else {
+            auto const& variable_token = std::get<VariableQueryToken>(logtype_token);
+            auto const variable_type = variable_token.get_variable_type();
+            auto const& raw_string = variable_token.get_query_substring();
+            auto const is_encoded_with_wildcard = variable_token.get_is_encoded_with_wildcard();
+            auto const var_has_wildcard = variable_token.get_has_wildcard();
+            auto& schema_type = lexer.m_id_symbol[variable_type];
+            encoded_variable_t encoded_var = 0;
+            if (is_encoded_with_wildcard) {
+                if (cIntVarName == schema_type) {
+                    LogTypeDictionaryEntry::add_int_var(m_logtype_string);
+                } else if (cFloatVarName == schema_type) {
+                    LogTypeDictionaryEntry::add_float_var(m_logtype_string);
+                }
+            } else if (false == var_has_wildcard && cIntVarName == schema_type
+                       && EncodedVariableInterpreter::convert_string_to_representable_integer_var(
+                               raw_string,
+                               encoded_var
+                       ))
+            {
+                LogTypeDictionaryEntry::add_int_var(m_logtype_string);
+            } else if (false == var_has_wildcard && cFloatVarName == schema_type
+                       && EncodedVariableInterpreter::convert_string_to_representable_float_var(
+                               raw_string,
+                               encoded_var
+                       ))
+            {
+                LogTypeDictionaryEntry::add_float_var(m_logtype_string);
+            } else {
+                LogTypeDictionaryEntry::add_dict_var(m_logtype_string);
+            }
+        }
+    }
+}
+
+auto QueryInterpretation::operator<(QueryInterpretation const& rhs) const -> bool {
+    if (m_logtype.size() < rhs.m_logtype.size()) {
+        return true;
+    }
+    if (m_logtype.size() > rhs.m_logtype.size()) {
+        return false;
+    }
+    for (uint32_t i = 0; i < m_logtype.size(); i++) {
+        if (m_logtype[i] < rhs.m_logtype[i]) {
+            return true;
+        }
+        if (m_logtype[i] > rhs.m_logtype[i]) {
+            return false;
+        }
+    }
+    return false;
+}
+
+auto operator<<(std::ostream& os, QueryInterpretation const& query_logtype) -> std::ostream& {
+    os << "logtype='";
+    for (uint32_t idx = 0; idx < query_logtype.get_logtype_size(); idx++) {
+        if (auto const& query_token = query_logtype.get_logtype_token(idx);
+            std::holds_alternative<StaticQueryToken>(query_token))
+        {
+            os << std::get<StaticQueryToken>(query_token).get_query_substring();
+        } else {
+            auto const& variable_token = std::get<VariableQueryToken>(query_token);
+            os << "<" << variable_token.get_variable_type() << ">("
+               << variable_token.get_query_substring() << ")";
+        }
+    }
+    os << "', has_wildcard='";
+    for (uint32_t idx = 0; idx < query_logtype.get_logtype_size(); idx++) {
+        if (auto const& query_token = query_logtype.get_logtype_token(idx);
+            std::holds_alternative<StaticQueryToken>(query_token))
+        {
+            os << 0;
+        } else {
+            auto const& variable_token = std::get<VariableQueryToken>(query_token);
+            os << variable_token.get_has_wildcard();
+        }
+    }
+    os << "', is_encoded_with_wildcard='";
+    for (uint32_t idx = 0; idx < query_logtype.get_logtype_size(); idx++) {
+        if (auto const& query_token = query_logtype.get_logtype_token(idx);
+            std::holds_alternative<StaticQueryToken>(query_token))
+        {
+            os << 0;
+        } else {
+            auto const& variable_token = std::get<VariableQueryToken>(query_token);
+            os << variable_token.get_is_encoded_with_wildcard();
+        }
+    }
+    os << "', logtype_string='" << query_logtype.get_logtype_string() << "'";
+    return os;
+}
+}  // namespace clp