Open datetime min value limit (apache#3158)

the min_value in olap/type.h of datetime is 0000-01-01 00:00:00, so we don't need restrict datetime min in tablet_sink
Ma1oneZhang · Mar 24, 2020 · d4c1938 · d4c1938
1 parent dff3c0d
commit d4c1938
Show file tree

Hide file tree

Showing 15 changed files with 82 additions and 52 deletions.
diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp
@@ -455,10 +455,6 @@ Status ScrollParser::fill_tuple(const TupleDescriptor* tuple_desc,
                     RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type);
                 }
 
-                if (ts_slot->year() < 1900) {
-                    RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type);
-                }
-
                 if (type == TYPE_DATE) {
                     ts_slot->cast_to_date();
                 } else {

diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp
@@ -822,27 +822,6 @@ int OlapTableSink::_validate_data(RuntimeState* state, RowBatch* batch, Bitmap*
                     LOG(INFO) << ss.str();
 #else
                     state->append_error_msg_to_file("", ss.str());
-#endif
-                    filtered_rows++;
-                    row_valid = false;
-                    filter_bitmap->Set(row_no, true);
-                    continue;
-                }
-                break;
-            }
-            case TYPE_DATE:
-            case TYPE_DATETIME: {
-                static DateTimeValue s_min_value = DateTimeValue(19000101000000UL);
-                // static DateTimeValue s_max_value = DateTimeValue(99991231235959UL);
-                DateTimeValue* date_val = (DateTimeValue*)slot;
-                if (*date_val < s_min_value) {
-                    std::stringstream ss;
-                    ss << "datetime value is not valid, column=" << desc->col_name()
-                        << ", value=" << date_val->debug_string();
-#if BE_TEST
-                    LOG(INFO) << ss.str();
-#else
-                    state->append_error_msg_to_file("", ss.str());
 #endif
                     filtered_rows++;
                     row_valid = false;

diff --git a/be/src/exec/text_converter.hpp b/be/src/exec/text_converter.hpp
@@ -129,11 +129,6 @@ inline bool TextConverter::write_slot(const SlotDescriptor* slot_desc,
             parse_result = StringParser::PARSE_FAILURE;
             break;
         }
-        // For compatibility with DPP, which only support years after 1900 
-        if (ts_slot->year() < 1900) {
-            parse_result = StringParser::PARSE_FAILURE;
-            break;
-        }
 
         ts_slot->cast_to_date();
         break;
@@ -144,11 +139,6 @@ inline bool TextConverter::write_slot(const SlotDescriptor* slot_desc,
         if (!ts_slot->from_date_str(data, len)) {
             parse_result = StringParser::PARSE_FAILURE;
         }
-        // For compatibility with DPP, which only support years after 1900 
-        if (ts_slot->year() < 1900) {
-            parse_result = StringParser::PARSE_FAILURE;
-            break;
-        }
 
         ts_slot->to_datetime();
         break;

diff --git a/be/test/olap/column_reader_test.cpp b/be/test/olap/column_reader_test.cpp
@@ -1780,6 +1780,69 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) {
     ASSERT_NE(_column_reader->next_vector(
         _col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS);
 }
+TEST_F(TestColumn, VectorizedDatetimeColumnZero) {
+    // write data
+    TabletSchema tablet_schema;
+
+    SetTabletSchemaWithOneColumn(
+            "DatetimeColumnWithoutPresent",
+            "DATETIME",
+            "REPLACE",
+            8,
+            true,
+            true, &tablet_schema);
+    CreateColumnWriter(tablet_schema);
+
+    RowCursor write_row;
+    write_row.init(tablet_schema);
+
+    RowBlock block(&tablet_schema);
+    RowBlockInfo block_info;
+    block_info.row_num = 10000;
+    block.init(block_info);
+
+    write_row.set_null(0);
+    block.set_row(0, write_row);
+    block.finalize(1);
+    ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS);
+
+    std::vector<string> val_string_array;
+    val_string_array.push_back("1000-01-01 00:00:00");
+    OlapTuple tuple(val_string_array);
+    write_row.from_tuple(tuple);
+    write_row.set_not_null(0);
+    block.set_row(0, write_row);
+    block.finalize(1);
+    ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS);
+
+    ColumnDataHeaderMessage header;
+    ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS);
+
+    // read data
+    CreateColumnReader(tablet_schema);
+
+    RowCursor read_row;
+    read_row.init(tablet_schema);
+
+    _col_vector.reset(new ColumnVector());
+    ASSERT_EQ(_column_reader->next_vector(
+            _col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS);
+    bool *is_null = _col_vector->is_null();
+    ASSERT_EQ(is_null[0], true);
+
+    char *data = reinterpret_cast<char *>(_col_vector->col_data());
+    ASSERT_EQ(is_null[1], false);
+
+    data += sizeof(uint64_t);
+    read_row.set_field_content(0, data, _mem_pool.get());
+    std::cout << read_row.to_string() << std::endl;
+    ASSERT_TRUE(strncmp(read_row.to_string().c_str(),
+            "0&1000-01-01 00:00:00", strlen("0&1000-01-01 00:00:00")) == 0);
+
+    ASSERT_NE(_column_reader->next_vector(
+            _col_vector.get(), 2, _mem_pool.get()), OLAP_SUCCESS);
+}
+
 
 TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) {
     // write data

diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
@@ -68,9 +68,9 @@ under the License.
             其中整数部分为 1 ~ 18
             不支持科学计数法
         DATE（3字节）
-            范围：1900-01-01 ~ 9999-12-31
+            范围：0000-01-01 ~ 9999-12-31
         DATETIME（8字节）
-            范围：1900-01-01 00:00:00 ~ 9999-12-31 23:59:59
+            范围：0000-01-01 00:00:00 ~ 9999-12-31 23:59:59
         CHAR[(length)]
             定长字符串。长度范围：1 ~ 255。默认为1
         VARCHAR[(length)]

diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATE.md
@@ -24,7 +24,7 @@ under the License.
             DATE(expr) 
         将输入的类型转化为DATE类型
     DATE类型
-        日期类型，目前的取值范围是['1900-01-01', '9999-12-31'], 默认的打印形式是'YYYY-MM-DD'
+        日期类型，目前的取值范围是['0000-01-01', '9999-12-31'], 默认的打印形式是'YYYY-MM-DD'
 
 ## example
     mysql> SELECT DATE('2003-12-31 01:02:03');

diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATETIME.md b/docs/documentation/cn/sql-reference/sql-statements/Data Types/DATETIME.md
@@ -20,7 +20,7 @@ under the License.
 # DATETIME
 ## description
     DATETIME
-    日期时间类型，取值范围是['1000-01-01 00:00:00', '9999-12-31 23:59:59'].
+    日期时间类型，取值范围是['0000-01-01 00:00:00', '9999-12-31 23:59:59'].
     打印的形式是'YYYY-MM-DD HH:MM:SS'
 
 ## keyword

diff --git a/...ocumentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md b/...ocumentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md
@@ -67,9 +67,9 @@ Syntax:
             fractional part: 0 ~ 9
             Not support scientific notation
         DATE(3 Bytes)
-            Range: 1900-01-01 ~ 9999-12-31
+            Range: 0000-01-01 ~ 9999-12-31
         DATETIME(8 Bytes)
-            Range: 1900-01-01 00:00:00 ~ 9999-12-31 23:59:59
+            Range: 0000-01-01 00:00:00 ~ 9999-12-31 23:59:59
         CHAR[(length)]
             Fixed length string. Range: 1 ~ 255. Default: 1
         VARCHAR[(length)]

diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/DATETIME_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/DATETIME_EN.md
@@ -20,7 +20,7 @@ under the License.
 # DATETIME
 ## Description
 DATETIME
-Date and time type, value range is ['1000-01-01 00:00:00','9999-12-31 23:59:59'].
+Date and time type, value range is ['0000-01-01 00:00:00','9999-12-31 23:59:59'].
 The form of printing is'YYYY-MM-DD HH:MM:SS'
 
 ##keyword

diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Types/DATE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Types/DATE_EN.md
@@ -24,7 +24,7 @@ Syntax:
 Date
 Convert input type to DATE type
 date
-Date type, the current range of values is ['1900-01-01','9999-12-31'], and the default print form is'YYYYY-MM-DD'.
+Date type, the current range of values is ['0000-01-01','9999-12-31'], and the default print form is'YYYYY-MM-DD'.
 
 ## example
 mysql> SELECT DATE('2003-12-31 01:02:03');

diff --git a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java
@@ -48,9 +48,9 @@
 public class DateLiteral extends LiteralExpr {
     private static final Logger LOG = LogManager.getLogger(DateLiteral.class);
 
-    private static final DateLiteral MIN_DATE = new DateLiteral(1900, 1, 1);
+    private static final DateLiteral MIN_DATE = new DateLiteral(0000, 1, 1);
     private static final DateLiteral MAX_DATE = new DateLiteral(9999, 12, 31);
-    private static final DateLiteral MIN_DATETIME = new DateLiteral(1900, 1, 1, 0, 0, 0);
+    private static final DateLiteral MIN_DATETIME = new DateLiteral(0000, 1, 1, 0, 0, 0);
     private static final DateLiteral MAX_DATETIME = new DateLiteral(9999, 12, 31, 23, 59, 59);
     public static final DateLiteral UNIX_EPOCH_TIME = new DateLiteral(1970, 01, 01, 00, 00, 00);
 

diff --git a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java
@@ -94,10 +94,10 @@ public class TimeUtils {
         TIME_FORMAT.setTimeZone(TIME_ZONE);
 
         try {
-            MIN_DATE = DATE_FORMAT.parse("1900-01-01");
+            MIN_DATE = DATE_FORMAT.parse("0000-01-01");
             MAX_DATE = DATE_FORMAT.parse("9999-12-31");
 
-            MIN_DATETIME = DATETIME_FORMAT.parse("1900-01-01 00:00:00");
+            MIN_DATETIME = DATETIME_FORMAT.parse("0000-01-01 00:00:00");
             MAX_DATETIME = DATETIME_FORMAT.parse("9999-12-31 23:59:59");
 
         } catch (ParseException e) {

diff --git a/fe/src/test/java/org/apache/doris/analysis/LiteralExprCompareTest.java b/fe/src/test/java/org/apache/doris/analysis/LiteralExprCompareTest.java
@@ -87,8 +87,8 @@ public void dateTest() throws AnalysisException {
         LiteralExpr minDatetime1Same = new DateLiteral(ScalarType.DATETIME, false);
         LiteralExpr date8 = new DateLiteral("9999-12-31", ScalarType.DATE);
         LiteralExpr date9 = new DateLiteral("9999-12-31 23:59:59", ScalarType.DATETIME);
-        LiteralExpr date10 = new DateLiteral("1900-01-01", ScalarType.DATE);
-        LiteralExpr date11 = new DateLiteral("1900-01-01 00:00:00", ScalarType.DATETIME);
+        LiteralExpr date10 = new DateLiteral("0000-01-01", ScalarType.DATE);
+        LiteralExpr date11 = new DateLiteral("0000-01-01 00:00:00", ScalarType.DATETIME);
 
         Assert.assertTrue(date1.equals(date1Same) && date1.compareLiteral(date1Same) == 0);
         Assert.assertTrue(date1.equals(date1Same) && date1.compareLiteral(date1Same) == 0);

diff --git a/fe/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java b/fe/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java
@@ -134,7 +134,7 @@ public void compareTest() throws AnalysisException {
         pk1 = PartitionKey.createPartitionKey(Arrays.asList(new PartitionValue("-128"), new PartitionValue("-32768"),
                 new PartitionValue("-2147483648"), new PartitionValue("-9223372036854775808"),
                 new PartitionValue("-170141183460469231731687303715884105728"),
-                new PartitionValue("1900-01-01"), new PartitionValue("1900-01-01 00:00:00")),
+                new PartitionValue("0000-01-01"), new PartitionValue("0000-01-01 00:00:00")),
                 allColumns);
         pk2 = PartitionKey.createInfinityPartitionKey(allColumns, false);
         Assert.assertTrue(pk1.equals(pk2) && pk1.compareTo(pk2) == 0);

diff --git a/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java b/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java
@@ -57,9 +57,9 @@ public void testNormal() {
         Assert.assertNotNull(TimeUtils.getStartTime());
         Assert.assertTrue(TimeUtils.getEstimatedTime(0L) > 0);
 
-        Assert.assertEquals(-2209017600000L, TimeUtils.MIN_DATE.getTime());
+        Assert.assertEquals(-62167420800000L, TimeUtils.MIN_DATE.getTime());
         Assert.assertEquals(253402185600000L, TimeUtils.MAX_DATE.getTime());
-        Assert.assertEquals(-2209017600000L, TimeUtils.MIN_DATETIME.getTime());
+        Assert.assertEquals(-62167420800000L, TimeUtils.MIN_DATETIME.getTime());
         Assert.assertEquals(253402271999000L, TimeUtils.MAX_DATETIME.getTime());
     }
 
@@ -74,6 +74,7 @@ public void testDateParse() {
         validDateList.add("9999-12-31");
         validDateList.add("1900-01-01");
         validDateList.add("2013-2-28");
+        validDateList.add("0000-01-01");
         for (String validDate : validDateList) {
             try {
                 TimeUtils.parseDate(validDate, PrimitiveType.DATE);
@@ -112,6 +113,7 @@ public void testDateParse() {
         validDateTimeList.add("2013-2-28 23:59:59");
         validDateTimeList.add("2013-2-28 2:3:4");
         validDateTimeList.add("2014-05-07 19:8:50");
+        validDateTimeList.add("0000-01-01 00:00:00");
         for (String validDateTime : validDateTimeList) {
             try {
                 TimeUtils.parseDate(validDateTime, PrimitiveType.DATETIME);