[Enhancement] upgrade roaring bitmap lib to v1.1.3 (#23502)

The related pr of RoaringBitmap: https://github.com/RoaringBitmap/CRoaring/pull/246/files This bug will cause queries based on BitmapIndex to return wrong data, so upgrade roaring bitmap lib Signed-off-by: trueeyu <lxhhust350@qq.com> (cherry picked from commit e6ee2f3) # Conflicts: # be/src/storage/rowset/segment_iterator.cpp # be/test/types/bitmap_value_test.cpp
StarRocks · May 17, 2023 · 015a260 · 015a260
1 parent f649e86
commit 015a260
Show file tree

Hide file tree

Showing 11 changed files with 404 additions and 10 deletions.
diff --git a/be/src/storage/del_vector.cpp b/be/src/storage/del_vector.cpp
@@ -109,7 +109,7 @@ string DelVector::to_string() const {
 void DelVector::_update_stats() {
     // TODO(cbl): optimization
     if (_roaring) {
-        roaring_statistics_t st;
+        roaring::api::roaring_statistics_t st;
         roaring_bitmap_statistics(&_roaring->roaring, &st);
         _memory_usage = st.n_bytes_array_containers + st.n_bytes_bitset_containers + st.n_bytes_run_containers;
         //_memory_usage = _roaring->getSizeInBytes(false);

diff --git a/be/src/storage/del_vector.h b/be/src/storage/del_vector.h
@@ -21,6 +21,8 @@
 
 namespace starrocks {
 
+using Roaring = roaring::Roaring;
+
 // A bitmap(uint32_t set) to store all the deleted rows' ids of a segment.
 // Each DelVector is associated with a version, which is EditVersion's majar version.
 // Serialization format:

diff --git a/be/src/storage/rowset/bitmap_index_reader.cpp b/be/src/storage/rowset/bitmap_index_reader.cpp
@@ -46,6 +46,8 @@
 
 namespace starrocks {
 
+using Roaring = roaring::Roaring;
+
 BitmapIndexReader::BitmapIndexReader() {
     MEM_TRACKER_SAFE_CONSUME(ExecEnv::GetInstance()->bitmap_index_mem_tracker(), sizeof(BitmapIndexReader));
 }

diff --git a/be/src/storage/rowset/bitmap_index_reader.h b/be/src/storage/rowset/bitmap_index_reader.h
@@ -52,6 +52,8 @@ class BitmapIndexIterator;
 class IndexedColumnReader;
 class IndexedColumnIterator;
 
+using Roaring = roaring::Roaring;
+
 class BitmapIndexReader {
 public:
     BitmapIndexReader();

diff --git a/be/src/storage/rowset/bitmap_index_writer.cpp b/be/src/storage/rowset/bitmap_index_writer.cpp
@@ -52,7 +52,7 @@
 
 namespace starrocks {
 
-namespace {
+using Roaring = roaring::Roaring;
 
 class BitmapUpdateContext {
     static const size_t estimate_size_threshold = 1024;
@@ -269,8 +269,6 @@ class BitmapIndexWriterImpl : public BitmapIndexWriter {
     mutable std::vector<BitmapUpdateContext*> _late_update_context_vector;
 };
 
-} // namespace
-
 struct BitmapIndexWriterBuilder {
     template <LogicalType ftype>
     std::unique_ptr<BitmapIndexWriter> operator()(const TypeInfoPtr& typeinfo) {

diff --git a/be/src/storage/rowset/bitmap_range_iterator.h b/be/src/storage/rowset/bitmap_range_iterator.h
@@ -76,15 +76,15 @@ class BitmapRangeIterator {
 
 private:
     void _read_next_batch() {
-        uint32_t n = roaring_read_uint32_iterator(&_iter, _buf, kBatchSize);
+        uint32_t n = roaring::api::roaring_read_uint32_iterator(&_iter, _buf, kBatchSize);
         _buf_pos = 0;
         _buf_size = n;
         _eof = n == 0;
     }
 
     static const uint32_t kBatchSize = 256;
 
-    roaring_uint32_iterator_t _iter{};
+    roaring::api::roaring_uint32_iterator_t _iter{};
     uint32_t _last_val{0};
     uint32_t _buf_pos{0};
     uint32_t _buf_size{0};

diff --git a/be/src/storage/rowset/segment_iterator.cpp b/be/src/storage/rowset/segment_iterator.cpp
@@ -258,7 +258,12 @@ class SegmentIterator final : public ChunkIterator {
 
     Status _get_del_vec_st;
     DelVectorPtr _del_vec;
+<<<<<<< HEAD
     roaring_uint32_iterator_t _roaring_iter;
+=======
+    DeltaColumnGroupList _dcgs;
+    roaring::api::roaring_uint32_iterator_t _roaring_iter;
+>>>>>>> e6ee2f353 ([Enhancement] upgrade roaring bitmap lib to v1.1.3 (#23502))
 
     std::unordered_map<ColumnId, std::unique_ptr<RandomAccessFile>> _column_files;
 

diff --git a/be/src/types/bitmap_value_detail.h b/be/src/types/bitmap_value_detail.h
@@ -90,6 +90,13 @@ struct BitmapTypeCode {
 
 namespace detail {
 
+// https://github.com/RoaringBitmap/CRoaring/blob/5d6dd2342d9e3ffaf481aa5ebe344e19984faa4a/src/roaring.c#L21
+// The tow macro is not in .h file, so copy to here.
+#define SERIALIZATION_ARRAY_UINT32 1
+#define SERIALIZATION_CONTAINER 2
+
+using Roaring = roaring::Roaring;
+
 class Roaring64MapSetBitForwardIterator;
 
 // Forked from https://github.com/RoaringBitmap/CRoaring/blob/v0.2.60/cpp/roaring64map.hh