Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Improve]delete match_element_xx and add fe config for create inverted index on array #34766

Merged
merged 3 commits into from
May 14, 2024
Merged

[Improve]delete match_element_xx and add fe config for create inverted index on array #34766

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2190f4b
delete match_element_xx and add fe config for create inverted index o…
amorynan May 13, 2024
137357e
fix weird code
amorynan May 13, 2024
b167ab9
add deprecated note instead of delete for thrift param
amorynan May 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
delete match_element_xx and add fe config for create inverted index o… 
…n array
  • Loading branch information
@amorynan
amorynan committed May 13, 2024
commit 2190f4b906d43f1a05371f2f967601a6086d8e38
10 changes: 0 additions & 10 deletions be/src/exec/olap_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,16 +318,6 @@ class ColumnValueRange {
condition.__set_condition_op("match_regexp");
} else if (value.first == MatchType::MATCH_PHRASE_EDGE) {
condition.__set_condition_op("match_phrase_edge");
} else if (value.first == MatchType::MATCH_ELEMENT_EQ) {
condition.__set_condition_op("match_element_eq");
} else if (value.first == MatchType::MATCH_ELEMENT_LT) {
condition.__set_condition_op("match_element_lt");
} else if (value.first == MatchType::MATCH_ELEMENT_GT) {
condition.__set_condition_op("match_element_gt");
} else if (value.first == MatchType::MATCH_ELEMENT_LE) {
condition.__set_condition_op("match_element_le");
} else if (value.first == MatchType::MATCH_ELEMENT_GE) {
condition.__set_condition_op("match_element_ge");
}
condition.condition_values.push_back(
cast_to_string<primitive_type, CppType>(value.second, _scale));
Expand Down
42 changes: 2 additions & 40 deletions be/src/exec/olap_utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,11 +132,6 @@ enum class MatchType {
MATCH_ANY = 0,
MATCH_ALL = 1,
MATCH_PHRASE = 2,
MATCH_ELEMENT_EQ = 3,
MATCH_ELEMENT_LT = 4,
MATCH_ELEMENT_GT = 5,
MATCH_ELEMENT_LE = 6,
MATCH_ELEMENT_GE = 7,
MATCH_PHRASE_PREFIX = 8,
MATCH_REGEXP = 9,
MATCH_PHRASE_EDGE = 10,
Expand All @@ -162,21 +157,6 @@ inline MatchType to_match_type(TExprOpcode::type type) {
case TExprOpcode::type::MATCH_PHRASE_EDGE:
return MatchType::MATCH_PHRASE_EDGE;
break;
case TExprOpcode::type::MATCH_ELEMENT_EQ:
return MatchType::MATCH_ELEMENT_EQ;
break;
case TExprOpcode::type::MATCH_ELEMENT_LT:
return MatchType::MATCH_ELEMENT_LT;
break;
case TExprOpcode::type::MATCH_ELEMENT_GT:
return MatchType::MATCH_ELEMENT_GT;
break;
case TExprOpcode::type::MATCH_ELEMENT_LE:
return MatchType::MATCH_ELEMENT_LE;
break;
case TExprOpcode::type::MATCH_ELEMENT_GE:
return MatchType::MATCH_ELEMENT_GE;
break;
default:
VLOG_CRITICAL << "TExprOpcode: " << type;
DCHECK(false);
Expand All @@ -197,16 +177,6 @@ inline MatchType to_match_type(const std::string& condition_op) {
return MatchType::MATCH_REGEXP;
} else if (condition_op.compare("match_phrase_edge") == 0) {
return MatchType::MATCH_PHRASE_EDGE;
} else if (condition_op.compare("match_element_eq") == 0) {
return MatchType::MATCH_ELEMENT_EQ;
} else if (condition_op.compare("match_element_lt") == 0) {
return MatchType::MATCH_ELEMENT_LT;
} else if (condition_op.compare("match_element_gt") == 0) {
return MatchType::MATCH_ELEMENT_GT;
} else if (condition_op.compare("match_element_le") == 0) {
return MatchType::MATCH_ELEMENT_LE;
} else if (condition_op.compare("match_element_ge") == 0) {
return MatchType::MATCH_ELEMENT_GE;
}
return MatchType::UNKNOWN;
}
Expand All @@ -216,12 +186,7 @@ inline bool is_match_condition(const std::string& op) {
0 == strcasecmp(op.c_str(), "match_phrase") ||
0 == strcasecmp(op.c_str(), "match_phrase_prefix") ||
0 == strcasecmp(op.c_str(), "match_regexp") ||
0 == strcasecmp(op.c_str(), "match_phrase_edge") ||
0 == strcasecmp(op.c_str(), "match_element_eq") ||
0 == strcasecmp(op.c_str(), "match_element_lt") ||
0 == strcasecmp(op.c_str(), "match_element_gt") ||
0 == strcasecmp(op.c_str(), "match_element_le") ||
0 == strcasecmp(op.c_str(), "match_element_ge")) {
0 == strcasecmp(op.c_str(), "match_phrase_edge")) {
return true;
}
return false;
Expand All @@ -230,10 +195,7 @@ inline bool is_match_condition(const std::string& op) {
inline bool is_match_operator(const TExprOpcode::type& op_type) {
return TExprOpcode::MATCH_ANY == op_type || TExprOpcode::MATCH_ALL == op_type ||
TExprOpcode::MATCH_PHRASE == op_type || TExprOpcode::MATCH_PHRASE_PREFIX == op_type ||
TExprOpcode::MATCH_REGEXP == op_type || TExprOpcode::MATCH_PHRASE_EDGE == op_type ||
TExprOpcode::MATCH_ELEMENT_EQ == op_type || TExprOpcode::MATCH_ELEMENT_LT == op_type ||
TExprOpcode::MATCH_ELEMENT_GT == op_type || TExprOpcode::MATCH_ELEMENT_LE == op_type ||
TExprOpcode::MATCH_ELEMENT_GE == op_type;
TExprOpcode::MATCH_REGEXP == op_type || TExprOpcode::MATCH_PHRASE_EDGE == op_type;
}

} // namespace doris
15 changes: 0 additions & 15 deletions be/src/olap/match_predicate.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,21 +116,6 @@ InvertedIndexQueryType MatchPredicate::_to_inverted_index_query_type(MatchType m
case MatchType::MATCH_PHRASE_EDGE:
ret = InvertedIndexQueryType::MATCH_PHRASE_EDGE_QUERY;
break;
case MatchType::MATCH_ELEMENT_EQ:
ret = InvertedIndexQueryType::EQUAL_QUERY;
break;
case MatchType::MATCH_ELEMENT_LT:
ret = InvertedIndexQueryType::LESS_THAN_QUERY;
break;
case MatchType::MATCH_ELEMENT_GT:
ret = InvertedIndexQueryType::GREATER_THAN_QUERY;
break;
case MatchType::MATCH_ELEMENT_LE:
ret = InvertedIndexQueryType::LESS_EQUAL_QUERY;
break;
case MatchType::MATCH_ELEMENT_GE:
ret = InvertedIndexQueryType::GREATER_EQUAL_QUERY;
break;
default:
DCHECK(false);
}
Expand Down
5 changes: 0 additions & 5 deletions be/src/vec/functions/match.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,11 +400,6 @@ void register_function_match(SimpleFunctionFactory& factory) {
factory.register_function<FunctionMatchPhrasePrefix>();
factory.register_function<FunctionMatchRegexp>();
factory.register_function<FunctionMatchPhraseEdge>();
factory.register_function<FunctionMatchElementEQ>();
factory.register_function<FunctionMatchElementLT>();
factory.register_function<FunctionMatchElementGT>();
factory.register_function<FunctionMatchElementLE>();
factory.register_function<FunctionMatchElementGE>();
}

} // namespace doris::vectorized
85 changes: 0 additions & 85 deletions be/src/vec/functions/match.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,89 +176,4 @@ class FunctionMatchPhraseEdge : public FunctionMatchBase {
}
};

class FunctionMatchElementEQ : public FunctionMatchBase {
public:
static constexpr auto name = "match_element_eq";
static FunctionPtr create() { return std::make_shared<FunctionMatchElementEQ>(); }

String get_name() const override { return name; }

Status execute_match(const std::string& column_name, const std::string& match_query_str,
size_t input_rows_count, const ColumnString* string_col,
InvertedIndexCtx* inverted_index_ctx,
const ColumnArray::Offsets64* array_offsets,
ColumnUInt8::Container& result) const override {
return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
"FunctionMatchElementEQ not support execute_match");
}
};

class FunctionMatchElementLT : public FunctionMatchBase {
public:
static constexpr auto name = "match_element_lt";
static FunctionPtr create() { return std::make_shared<FunctionMatchElementLT>(); }

String get_name() const override { return name; }

Status execute_match(const std::string& column_name, const std::string& match_query_str,
size_t input_rows_count, const ColumnString* string_col,
InvertedIndexCtx* inverted_index_ctx,
const ColumnArray::Offsets64* array_offsets,
ColumnUInt8::Container& result) const override {
return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
"FunctionMatchElementLT not support execute_match");
}
};

class FunctionMatchElementGT : public FunctionMatchBase {
public:
static constexpr auto name = "match_element_gt";
static FunctionPtr create() { return std::make_shared<FunctionMatchElementGT>(); }

String get_name() const override { return name; }

Status execute_match(const std::string& column_name, const std::string& match_query_str,
size_t input_rows_count, const ColumnString* string_col,
InvertedIndexCtx* inverted_index_ctx,
const ColumnArray::Offsets64* array_offsets,
ColumnUInt8::Container& result) const override {
return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
"FunctionMatchElementGT not support execute_match");
}
};

class FunctionMatchElementLE : public FunctionMatchBase {
public:
static constexpr auto name = "match_element_le";
static FunctionPtr create() { return std::make_shared<FunctionMatchElementLE>(); }

String get_name() const override { return name; }

Status execute_match(const std::string& column_name, const std::string& match_query_str,
size_t input_rows_count, const ColumnString* string_col,
InvertedIndexCtx* inverted_index_ctx,
const ColumnArray::Offsets64* array_offsets,
ColumnUInt8::Container& result) const override {
return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
"FunctionMatchElementLE not support execute_match");
}
};

class FunctionMatchElementGE : public FunctionMatchBase {
public:
static constexpr auto name = "match_element_ge";
static FunctionPtr create() { return std::make_shared<FunctionMatchElementGE>(); }

String get_name() const override { return name; }

Status execute_match(const std::string& column_name, const std::string& match_query_str,
size_t input_rows_count, const ColumnString* string_col,
InvertedIndexCtx* inverted_index_ctx,
const ColumnArray::Offsets64* array_offsets,
ColumnUInt8::Container& result) const override {
return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
"FunctionMatchElementGE not support execute_match");
}
};

} // namespace doris::vectorized
3 changes: 3 additions & 0 deletions fe/fe-common/src/main/java/org/apache/doris/common/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2694,6 +2694,9 @@ public static boolean isNotCloudMode() {
@ConfField(mutable = true, masterOnly = true)
public static boolean enable_create_bitmap_index_as_inverted_index = false;

@ConfField(mutable = true)
public static boolean enable_create_inverted_index_for_array = false;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If array_contains is OK, keep it true and disable index def with parser

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe we also need it for 2.1 because here depends on this pr: #32620 enable expr push down storage layer to pass inverted index , but this feature is not gonna to be in 2.1


// The original meta read lock is not enough to keep a snapshot of partition versions,
// so the execution of `createScanRangeLocations` are delayed to `Coordinator::exec`,
// to help to acquire a snapshot of partition versions.
Expand Down
5 changes: 0 additions & 5 deletions fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -351,11 +351,6 @@ MAP: 'MAP';
MATCH: 'MATCH';
MATCH_ALL: 'MATCH_ALL';
MATCH_ANY: 'MATCH_ANY';
MATCH_ELEMENT_EQ: 'ELEMENT_EQ';
MATCH_ELEMENT_GE: 'ELEMENT_GE';
MATCH_ELEMENT_GT: 'ELEMENT_GT';
MATCH_ELEMENT_LE: 'ELEMENT_LE';
MATCH_ELEMENT_LT: 'ELEMENT_LT';
MATCH_PHRASE: 'MATCH_PHRASE';
MATCH_PHRASE_EDGE: 'MATCH_PHRASE_EDGE';
MATCH_PHRASE_PREFIX: 'MATCH_PHRASE_PREFIX';
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1168,7 +1168,7 @@ nonReserved
| MAP
| MATCH_ALL
| MATCH_ANY
| MATCH_ELEMENT_EQ
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

delete others

| MATCH_columnName
| MATCH_ELEMENT_GE
| MATCH_ELEMENT_GT
| MATCH_ELEMENT_LE
Expand Down
27 changes: 1 addition & 26 deletions fe/fe-core/src/main/cup/sql_parser.cup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -493,11 +493,6 @@ terminal String
KW_MATCH_PHRASE_PREFIX,
KW_MATCH_REGEXP,
KW_MATCH_PHRASE_EDGE,
KW_MATCH_ELEMENT_EQ,
KW_MATCH_ELEMENT_LT,
KW_MATCH_ELEMENT_GT,
KW_MATCH_ELEMENT_LE,
KW_MATCH_ELEMENT_GE,
KW_NAME,
KW_NAMES,
KW_NATURAL,
Expand Down Expand Up @@ -1019,7 +1014,7 @@ precedence left KW_AND;
precedence left KW_NOT, NOT;
precedence left KW_BETWEEN, KW_IN, KW_IS, KW_EXISTS;
precedence left KW_LIKE, KW_REGEXP;
precedence left KW_MATCH_ANY, KW_MATCH_ALL, KW_MATCH_PHRASE, KW_MATCH_PHRASE_PREFIX, KW_MATCH_REGEXP, KW_MATCH_PHRASE_EDGE, KW_MATCH, KW_MATCH_ELEMENT_EQ, KW_MATCH_ELEMENT_LT, KW_MATCH_ELEMENT_GT, KW_MATCH_ELEMENT_LE, KW_MATCH_ELEMENT_GE;
precedence left KW_MATCH_ANY, KW_MATCH_ALL, KW_MATCH_PHRASE, KW_MATCH_PHRASE_PREFIX, KW_MATCH_REGEXP, KW_MATCH_PHRASE_EDGE, KW_MATCH;
precedence left EQUAL, LESSTHAN, GREATERTHAN;
precedence left ADD, SUBTRACT;
precedence left AT, STAR, DIVIDE, MOD, KW_DIV;
Expand Down Expand Up @@ -7565,16 +7560,6 @@ match_predicate ::=
{: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_REGEXP, e1, e2); :}
| expr:e1 KW_MATCH_PHRASE_EDGE expr:e2
{: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_PHRASE_EDGE, e1, e2); :}
| expr:e1 KW_MATCH_ELEMENT_EQ expr:e2
{: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_ELEMENT_EQ, e1, e2); :}
| expr:e1 KW_MATCH_ELEMENT_LT expr:e2
{: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_ELEMENT_LT, e1, e2); :}
| expr:e1 KW_MATCH_ELEMENT_GT expr:e2
{: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_ELEMENT_GT, e1, e2); :}
| expr:e1 KW_MATCH_ELEMENT_LE expr:e2
{: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_ELEMENT_LE, e1, e2); :}
| expr:e1 KW_MATCH_ELEMENT_GE expr:e2
{: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_ELEMENT_GE, e1, e2); :}
;

// Avoid a reduce/reduce conflict with compound_predicate by explicitly
Expand Down Expand Up @@ -8443,16 +8428,6 @@ keyword ::=
{: RESULT = id; :}
| KW_MATCH_PHRASE_EDGE:id
{: RESULT = id; :}
| KW_MATCH_ELEMENT_EQ:id
{: RESULT = id; :}
| KW_MATCH_ELEMENT_LT:id
{: RESULT = id; :}
| KW_MATCH_ELEMENT_GT:id
{: RESULT = id; :}
| KW_MATCH_ELEMENT_LE:id
{: RESULT = id; :}
| KW_MATCH_ELEMENT_GE:id
{: RESULT = id; :}
;

// Identifier that contain keyword
Expand Down
4 changes: 4 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;

import com.google.common.base.Strings;
import com.google.common.collect.Lists;
Expand Down Expand Up @@ -237,6 +238,9 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe
}

if (indexType == IndexType.INVERTED) {
if (!Config.enable_create_inverted_index_for_array && colType.isArrayType()) {
throw new AnalysisException("inverted index does not support array type column:" + indexColName);
}
InvertedIndexUtil.checkInvertedIndexParser(indexColName, colType, properties);
} else if (indexType == IndexType.NGRAM_BF) {
if (colType != PrimitiveType.CHAR && colType != PrimitiveType.VARCHAR
Expand Down
Loading
Loading