Support boolean/date/datetime/decimal types in segment V2 (apache#1863)

Ma1oneZhang · Sep 25, 2019 · eb840ec · eb840ec
1 parent c643cbd
commit eb840ec
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 0 deletions.
diff --git a/be/src/olap/rowset/segment_v2/bitshuffle_page.h b/be/src/olap/rowset/segment_v2/bitshuffle_page.h
@@ -232,8 +232,10 @@ class BitShufflePageDecoder : public PageDecoder {
         switch (_size_of_element) {
             case 1:
             case 2:
+            case 3:
             case 4:
             case 8:
+            case 12:
             case 16:
                 break;
             default:

diff --git a/be/src/olap/rowset/segment_v2/encoding_info.cpp b/be/src/olap/rowset/segment_v2/encoding_info.cpp
@@ -19,6 +19,7 @@
 
 #include "olap/olap_common.h"
 #include "olap/rowset/segment_v2/bitshuffle_page.h"
+#include "olap/rowset/segment_v2/rle_page.h"
 
 namespace doris {
 namespace segment_v2 {
@@ -54,6 +55,18 @@ struct TypeEncodingTraits<type, BIT_SHUFFLE> {
     }
 };
 
+template<FieldType type>
+struct TypeEncodingTraits<type, RLE> {
+    static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
+        *builder = new RlePageBuilder<type>(opts);
+        return Status::OK();
+    }
+    static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts, PageDecoder** decoder) {
+        *decoder = new RlePageDecoder<type>(data, opts);
+        return Status::OK();
+    }
+};
+
 template<FieldType Type, EncodingTypePB Encoding>
 struct EncodingTraits : TypeEncodingTraits<Type, Encoding> {
     static const FieldType type = Type;
@@ -109,6 +122,15 @@ EncodingInfoResolver::EncodingInfoResolver() {
     _add_map<OLAP_FIELD_TYPE_FLOAT, PLAIN_ENCODING>();
     _add_map<OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE>();
     _add_map<OLAP_FIELD_TYPE_DOUBLE, PLAIN_ENCODING>();
+    _add_map<OLAP_FIELD_TYPE_BOOL, RLE>();
+    _add_map<OLAP_FIELD_TYPE_BOOL, BIT_SHUFFLE>();
+    _add_map<OLAP_FIELD_TYPE_BOOL, PLAIN_ENCODING>();
+    _add_map<OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE>();
+    _add_map<OLAP_FIELD_TYPE_DATE, PLAIN_ENCODING>();
+    _add_map<OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>();
+    _add_map<OLAP_FIELD_TYPE_DATETIME, PLAIN_ENCODING>();
+    _add_map<OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE>();
+    _add_map<OLAP_FIELD_TYPE_DECIMAL, PLAIN_ENCODING>();
 }
 
 EncodingInfoResolver::~EncodingInfoResolver() {

diff --git a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
@@ -17,6 +17,7 @@
 
 #include "olap/rowset/segment_v2/column_reader.h"
 #include "olap/rowset/segment_v2/column_writer.h"
+#include "olap/decimal12.h"
 
 #include <gtest/gtest.h>
 #include <iostream>
@@ -201,6 +202,41 @@ TEST_F(ColumnReaderWriterTest, test_nullable) {
     delete[] double_vals;
 }
 
+TEST_F(ColumnReaderWriterTest, test_types) {
+    size_t num_uint8_rows = 1024 * 1024;
+    uint8_t* is_null = new uint8_t[num_uint8_rows];
+
+    bool* bool_vals = new bool[num_uint8_rows];
+    uint24_t* date_vals = new uint24_t[num_uint8_rows];
+    uint64_t* datetime_vals = new uint64_t[num_uint8_rows];
+    decimal12_t* decimal_vals = new decimal12_t[num_uint8_rows];
+    for (int i = 0; i < num_uint8_rows; ++i) {
+        bool_vals[i] = i % 2;
+        date_vals[i] = i + 33;
+        datetime_vals[i] = i + 33;
+        decimal_vals[i] = decimal12_t(i, i); // 1.000000001
+        BitmapChange(is_null, i, (i % 4) == 0);
+    }
+    test_nullable_data<OLAP_FIELD_TYPE_BOOL, BIT_SHUFFLE>((uint8_t*)bool_vals, is_null, num_uint8_rows, "null_bool_bs");
+    test_nullable_data<OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE>((uint8_t*)date_vals, is_null, num_uint8_rows / 3, "null_date_bs");
+
+    for (int i = 0; i < num_uint8_rows; ++i) {
+        BitmapChange(is_null, i, (i % 16) == 0);
+    }
+    test_nullable_data<OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>((uint8_t*)datetime_vals, is_null, num_uint8_rows / 8, "null_datetime_bs");
+
+    for (int i = 0; i< num_uint8_rows; ++i) {
+        BitmapChange(is_null, i, (i % 24) == 0);
+    }
+    test_nullable_data<OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE>((uint8_t*)decimal_vals, is_null, num_uint8_rows / 12, "null_decimal_bs");
+
+    delete[] is_null;
+    delete[] bool_vals;
+    delete[] date_vals;
+    delete[] datetime_vals;
+    delete[] decimal_vals;
+}
+
 }
 }