[Optimize](index) Optimize implement the new internal lucene index qu…

…ery interface (apache#23389)
w41ter · Aug 25, 2023 · 2847c5e · 2847c5e
1 parent d30bb80
commit 2847c5e
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 13 deletions.
diff --git a/be/src/clucene b/be/src/clucene
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -294,12 +294,28 @@ Status FullTextIndexReader::query(OlapReaderStatistics* stats, const std::string
             }
 
             try {
-                SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer);
-                index_searcher->_search(query.get(), [&term_match_bitmap](const int32_t docid,
-                                                                          const float_t /*score*/) {
-                    // docid equal to rowid in segment
-                    term_match_bitmap->add(docid);
-                });
+                if (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
+                    query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
+                    query_type == InvertedIndexQueryType::EQUAL_QUERY) {
+                    SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer);
+                    index_searcher->_search(query.get(), [&term_match_bitmap](DocRange* docRange) {
+                        if (docRange->type_ == DocRangeType::kMany) {
+                            term_match_bitmap->addMany(docRange->doc_many_size_,
+                                                       docRange->doc_many.data());
+                        } else {
+                            term_match_bitmap->addRange(docRange->doc_range.first,
+                                                        docRange->doc_range.second);
+                        }
+                    });
+                } else {
+                    SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer);
+                    index_searcher->_search(
+                            query.get(),
+                            [&term_match_bitmap](const int32_t docid, const float_t /*score*/) {
+                                // docid equal to rowid in segment
+                                term_match_bitmap->add(docid);
+                            });
+                }
             } catch (const CLuceneError& e) {
                 return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
                         "CLuceneError occured: {}", e.what());
@@ -518,12 +534,25 @@ Status StringTypeInvertedIndexReader::query(OlapReaderStatistics* stats,
     read_null_bitmap(&null_bitmap_cache_handle, index_searcher->getReader()->directory());
 
     try {
-        SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer);
-        index_searcher->_search(query.get(),
-                                [&result](const int32_t docid, const float_t /*score*/) {
-                                    // docid equal to rowid in segment
-                                    result.add(docid);
-                                });
+        if (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
+            query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
+            query_type == InvertedIndexQueryType::EQUAL_QUERY) {
+            SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer);
+            index_searcher->_search(query.get(), [&result](DocRange* docRange) {
+                if (docRange->type_ == DocRangeType::kMany) {
+                    result.addMany(docRange->doc_many_size_, docRange->doc_many.data());
+                } else {
+                    result.addRange(docRange->doc_range.first, docRange->doc_range.second);
+                }
+            });
+        } else {
+            SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer);
+            index_searcher->_search(query.get(),
+                                    [&result](const int32_t docid, const float_t /*score*/) {
+                                        // docid equal to rowid in segment
+                                        result.add(docid);
+                                    });
+        }
     } catch (const CLuceneError& e) {
         if (_is_range_query(query_type) && e.number() == CL_ERR_TooManyClauses) {
             return Status::Error<ErrorCode::INVERTED_INDEX_BYPASS>(
+29 −0		src/core/CLucene/index/DocRange.h
+28 −0		src/core/CLucene/index/MultiSegmentReader.cpp
+4 −0		src/core/CLucene/index/MultipleTermPositions.cpp
+1 −0		src/core/CLucene/index/MultipleTermPositions.h
+80 −0		src/core/CLucene/index/SegmentTermDocs.cpp
+4 −0		src/core/CLucene/index/SegmentTermPositions.cpp
+3 −0		src/core/CLucene/index/Terms.h
+1 −0		src/core/CLucene/index/_MultiSegmentReader.h
+2 −0		src/core/CLucene/index/_SegmentHeader.h
+47 −8		src/core/CLucene/search/IndexSearcher.cpp
+3 −0		src/core/CLucene/search/IndexSearcher.h
+7 −0		src/core/CLucene/search/Scorer.cpp
+5 −1		src/core/CLucene/search/Scorer.h
+3 −0		src/core/CLucene/search/SearchHeader.h
+4 −0		src/core/CLucene/search/TermScorer.cpp
+1 −0		src/core/CLucene/search/_TermScorer.h
+2 −1		src/test/CMakeLists.txt
+1,109 −0		src/test/search/TestSearchRange.cpp
+1 −0		src/test/test.h
+2 −1		src/test/tests.cpp
+0 −0		src/test/util/TestStrConvert.cpp