From 3fa2330020fe44f90e118ca774efb749373c49df Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Wed, 12 Oct 2022 11:22:18 +0800 Subject: [PATCH 1/7] refactor(query): remove precision in timestamp type --- .../20-data-type-time-date-types.md | 108 +++++++++--------- .../type_conversion.md | 2 +- .../30-sql/70-system-tables/system-columns.md | 2 +- .../70-system-tables/system-query-log.md | 6 +- .../src/data_from_to_protobuf_impl.rs | 4 +- src/meta/proto-conv/src/util.rs | 1 + src/meta/proto-conv/tests/it/proto_conv.rs | 6 +- src/meta/protos/proto/datatype.proto | 6 +- src/query/ast/src/ast/expr.rs | 11 +- src/query/ast/src/parser/expr.rs | 4 +- src/query/ast/tests/it/testdata/statement.txt | 12 +- src/query/datablocks/tests/it/data_block.rs | 4 +- src/query/datavalues/src/types/data_type.rs | 24 +--- .../src/types/deserializations/timestamp.rs | 1 - src/query/datavalues/src/types/eq.rs | 13 +-- .../src/types/serializations/timestamp.rs | 16 +-- .../datavalues/src/types/type_coercion.rs | 11 +- .../datavalues/src/types/type_factory.rs | 6 +- .../datavalues/src/types/type_timestamp.rs | 43 ++----- src/query/datavalues/src/types/type_traits.rs | 2 +- .../tests/it/types/create_column.rs | 4 +- .../tests/it/types/serializations/mod.rs | 4 +- src/query/expression/src/converts/from.rs | 4 +- src/query/expression/tests/it/convert.rs | 2 +- .../src/scalars/dates/interval_function.rs | 16 +-- src/query/functions/src/scalars/dates/now.rs | 2 +- .../src/scalars/dates/number_function.rs | 4 +- .../src/scalars/dates/round_function.rs | 2 +- .../expressions/cast_from_datetimes.rs | 2 +- .../functions/src/scalars/expressions/ctx.rs | 11 +- .../tests/it/aggregates/aggregate_function.rs | 4 +- .../functions/tests/it/scalars/arithmetics.rs | 10 +- .../functions/tests/it/scalars/dates/date.rs | 4 +- .../tests/it/scalars/dates/date_function.rs | 44 +++---- .../it/scalars/dates/interval_function.rs | 10 +- .../functions/tests/it/scalars/expressions.rs | 4 +- .../it/scalars/others/running_difference.rs | 10 +- .../tests/it/monotonicity.rs | 4 +- src/query/legacy-parser/src/sql_common.rs | 25 +--- .../service/src/sql/planner/binder/table.rs | 2 +- .../src/sql/planner/semantic/type_check.rs | 32 +++--- .../table_functions/fuse_blocks/fuse_block.rs | 2 +- .../fuse_snapshots/fuse_snapshot.rs | 2 +- src/query/storages/hive/src/converters.rs | 2 +- .../src/system/clustering_history_table.rs | 4 +- .../preludes/src/system/query_log_table.rs | 4 +- .../02_function/02_0012_function_datetimes | 36 +++--- .../base/05_ddl/05_0000_ddl_create_tables | 8 +- website/blog/2022-09-14-json-datatype.md | 56 ++++----- 49 files changed, 232 insertions(+), 364 deletions(-) diff --git a/docs/doc/30-reference/10-data-types/20-data-type-time-date-types.md b/docs/doc/30-reference/10-data-types/20-data-type-time-date-types.md index 1b99569d426f..0f5ced48e13d 100644 --- a/docs/doc/30-reference/10-data-types/20-data-type-time-date-types.md +++ b/docs/doc/30-reference/10-data-types/20-data-type-time-date-types.md @@ -17,30 +17,30 @@ See [Date & Time Functions](/doc/reference/functions/datetime-functions). ## Example ```sql -CREATE TABLE test_dt - ( - date DATE, - ts TIMESTAMP - ); +CREATE TABLE test_dt + ( + date DATE, + ts TIMESTAMP + ); DESC test_dt; +-------+--------------+------+---------+-------+ | Field | Type | Null | Default | Extra | +-------+--------------+------+---------+-------+ | date | DATE | NO | 0 | | -| ts | TIMESTAMP(6) | NO | 0 | | +| ts | TIMESTAMP | NO | 0 | | +-------+--------------+------+---------+-------+ -- A TIMESTAMP value can optionally include a trailing fractional seconds part in up to microseconds (6 digits) precision. -INSERT INTO test_dt -VALUES ('2022-04-07', - '2022-04-07 01:01:01.123456'), - ('2022-04-08', - '2022-04-08 01:01:01'); +INSERT INTO test_dt +VALUES ('2022-04-07', + '2022-04-07 01:01:01.123456'), + ('2022-04-08', + '2022-04-08 01:01:01'); -SELECT * -FROM test_dt; +SELECT * +FROM test_dt; +------------+----------------------------+ | date | ts | +------------+----------------------------+ @@ -50,24 +50,24 @@ FROM test_dt; -- Databend recognizes TIMESTAMP values in several formats. -CREATE TABLE test_formats - ( - id INT, - a TIMESTAMP - ); - -INSERT INTO test_formats -VALUES (1, - '2022-01-01 02:00:11'), - (2, - '2022-01-02T02:00:22'), - (3, - '2022-02-02T04:00:03+00:00'), - (4, - '2022-02-03'); - -SELECT * -FROM test_formats; +CREATE TABLE test_formats + ( + id INT, + a TIMESTAMP + ); + +INSERT INTO test_formats +VALUES (1, + '2022-01-01 02:00:11'), + (2, + '2022-01-02T02:00:22'), + (3, + '2022-02-02T04:00:03+00:00'), + (4, + '2022-02-03'); + +SELECT * +FROM test_formats; ---- 1 2022-01-01 02:00:11.000000 @@ -77,30 +77,30 @@ FROM test_formats; -- Databend automatically adjusts and shows TIMESTAMP values based on your current timezone. -CREATE TABLE test_tz - ( - id INT, - t TIMESTAMP - ); +CREATE TABLE test_tz + ( + id INT, + t TIMESTAMP + ); SET timezone='UTC'; -INSERT INTO test_tz -VALUES (1, - '2022-02-03T03:00:00'), - (2, - '2022-02-03T03:00:00+08:00'), - (3, - '2022-02-03T03:00:00-08:00'), - (4, - '2022-02-03'), - (5, - '2022-02-03T03:00:00+09:00'), - (6, - '2022-02-03T03:00:00+06:00'); - -SELECT * -FROM test_tz; +INSERT INTO test_tz +VALUES (1, + '2022-02-03T03:00:00'), + (2, + '2022-02-03T03:00:00+08:00'), + (3, + '2022-02-03T03:00:00-08:00'), + (4, + '2022-02-03'), + (5, + '2022-02-03T03:00:00+09:00'), + (6, + '2022-02-03T03:00:00+06:00'); + +SELECT * +FROM test_tz; ---- 1 2022-02-03 03:00:00.000000 @@ -112,8 +112,8 @@ FROM test_tz; SET timezone='Asia/Shanghai'; -SELECT * -FROM test_tz; +SELECT * +FROM test_tz; ---- 1 2022-02-03 11:00:00.000000 diff --git a/docs/doc/30-reference/20-functions/60-conversion-functions/type_conversion.md b/docs/doc/30-reference/20-functions/60-conversion-functions/type_conversion.md index 818b4b89ce9d..dee9d17da978 100644 --- a/docs/doc/30-reference/20-functions/60-conversion-functions/type_conversion.md +++ b/docs/doc/30-reference/20-functions/60-conversion-functions/type_conversion.md @@ -2,7 +2,7 @@ title: Type Conversion --- -Type conversion to target type. +Type conversion to target type. ## Syntax diff --git a/docs/doc/30-reference/30-sql/70-system-tables/system-columns.md b/docs/doc/30-reference/30-sql/70-system-tables/system-columns.md index fded23e3f4f9..6f3c050bff7a 100644 --- a/docs/doc/30-reference/30-sql/70-system-tables/system-columns.md +++ b/docs/doc/30-reference/30-sql/70-system-tables/system-columns.md @@ -58,7 +58,7 @@ SELECT * FROM system.columns WHERE database='system'; | query_kind | system | query_log | VARCHAR | | | 0 | | | query_text | system | query_log | VARCHAR | | | 0 | | | event_date | system | query_log | DATE | | | 0 | | -| event_time | system | query_log | TIMESTAMP(3) | | | 0 | | +| event_time | system | query_log | TIMESTAMP | | | 0 | | | current_database | system | query_log | VARCHAR | | | 0 | | | databases | system | query_log | VARCHAR | | | 0 | | | tables | system | query_log | VARCHAR | | | 0 | | diff --git a/docs/doc/30-reference/30-sql/70-system-tables/system-query-log.md b/docs/doc/30-reference/30-sql/70-system-tables/system-query-log.md index c42f94bb18fd..436ab24103ac 100644 --- a/docs/doc/30-reference/30-sql/70-system-tables/system-query-log.md +++ b/docs/doc/30-reference/30-sql/70-system-tables/system-query-log.md @@ -20,7 +20,7 @@ CREATE TABLE `query_log` ( `query_kind` VARCHAR, `query_text` VARCHAR, `event_date` DATE, - `event_time` TIMESTAMP(3), + `event_time` TIMESTAMP, `current_database` VARCHAR, `databases` VARCHAR, `tables` VARCHAR, @@ -85,8 +85,8 @@ written_io_bytes_cost_ms: 0 , skip_header=0, sql_dialect=PostgreSQL, storage_read_buffer_size=1048576, timezone=UTC, unquoted_ident_case_sensitive=0, wait_for_async_insert=1, wait_for_async_insert_timeout=100, scope: SESSION extra: - - + + *************************** 5. row *************************** log_type: 2 handler_type: MySQL diff --git a/src/meta/proto-conv/src/data_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/data_from_to_protobuf_impl.rs index 6bfc55dc1941..3bb46af5df99 100644 --- a/src/meta/proto-conv/src/data_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/data_from_to_protobuf_impl.rs @@ -359,7 +359,7 @@ impl FromToProto for dv::TimestampType { fn from_pb(p: pb::Timestamp) -> Result where Self: Sized { check_ver(p.ver, p.min_compatible)?; - let v = dv::TimestampType::create(p.precision as usize); + let v = dv::TimestampType::create(); Ok(v) } @@ -367,8 +367,6 @@ impl FromToProto for dv::TimestampType { let p = pb::Timestamp { ver: VER, min_compatible: MIN_COMPATIBLE_VER, - precision: self.precision() as u64, - // tz: self.tz().cloned(), }; Ok(p) diff --git a/src/meta/proto-conv/src/util.rs b/src/meta/proto-conv/src/util.rs index bc69dca51bc3..275a3913cf8d 100644 --- a/src/meta/proto-conv/src/util.rs +++ b/src/meta/proto-conv/src/util.rs @@ -61,6 +61,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[ 14, "2022-10-11: Add: role_arn and external_id in config.proto/OssStorageConfig, Remove role_arn and oidc_token from config.proto/OssStorageConfig", ), + (15, "2022-10-12: Remove: precision in TimestampType"), ]; pub const VER: u64 = META_CHANGE_LOG.last().unwrap().0; diff --git a/src/meta/proto-conv/tests/it/proto_conv.rs b/src/meta/proto-conv/tests/it/proto_conv.rs index 11480949e6e1..c1b26e641461 100644 --- a/src/meta/proto-conv/tests/it/proto_conv.rs +++ b/src/meta/proto-conv/tests/it/proto_conv.rs @@ -166,7 +166,7 @@ fn new_table_meta() -> mt::TableMeta { dv::DataField::new("float32", dv::Float32Type::default().into()), dv::DataField::new("float64", dv::Float64Type::default().into()), dv::DataField::new("date", dv::DateType::default().into()), - dv::DataField::new("timestamp", dv::TimestampType::create(5).into()), + dv::DataField::new("timestamp", dv::TimestampType::create().into()), dv::DataField::new("string", dv::StringType::default().into()), dv::DataField::new( "struct", @@ -232,7 +232,7 @@ fn new_table_meta_v10() -> mt::TableMeta { dv::DataField::new("float32", dv::Float32Type::default().into()), dv::DataField::new("float64", dv::Float64Type::default().into()), dv::DataField::new("date", dv::DateType::default().into()), - dv::DataField::new("timestamp", dv::TimestampType::create(5).into()), + dv::DataField::new("timestamp", dv::TimestampType::create().into()), dv::DataField::new("string", dv::StringType::default().into()), dv::DataField::new( "struct", @@ -298,7 +298,7 @@ fn new_table_meta_v12() -> mt::TableMeta { dv::DataField::new("float32", dv::Float32Type::default().into()), dv::DataField::new("float64", dv::Float64Type::default().into()), dv::DataField::new("date", dv::DateType::default().into()), - dv::DataField::new("timestamp", dv::TimestampType::create(5).into()), + dv::DataField::new("timestamp", dv::TimestampType::create().into()), dv::DataField::new("string", dv::StringType::default().into()), dv::DataField::new( "struct", diff --git a/src/meta/protos/proto/datatype.proto b/src/meta/protos/proto/datatype.proto index f48dd4f26814..9e9a59e71752 100644 --- a/src/meta/protos/proto/datatype.proto +++ b/src/meta/protos/proto/datatype.proto @@ -61,10 +61,8 @@ message Timestamp { uint64 ver = 100; uint64 min_compatible = 101; - /// The time resolution is determined by the precision parameter, range from 0 - /// to 9 Typically are used - 0 (seconds) 3 (milliseconds), 6 (microseconds), - /// 9 (nanoseconds). - uint64 precision = 1; + /// It's deprecated + reserved 1; } // Struct is similar to a `map` with fixed keys. diff --git a/src/query/ast/src/ast/expr.rs b/src/query/ast/src/ast/expr.rs index 8db02adec11b..88ded21bc4aa 100644 --- a/src/query/ast/src/ast/expr.rs +++ b/src/query/ast/src/ast/expr.rs @@ -242,9 +242,7 @@ pub enum TypeName { Float32, Float64, Date, - Timestamp { - precision: Option, - }, + Timestamp, String, Array { item_type: Option>, @@ -512,11 +510,8 @@ impl Display for TypeName { TypeName::Date => { write!(f, "DATE")?; } - TypeName::Timestamp { precision } => { - write!(f, "Timestamp")?; - if let Some(precision) = precision { - write!(f, "({})", *precision)?; - } + TypeName::Timestamp => { + write!(f, "TIMESTAMP")?; } TypeName::String => { write!(f, "STRING")?; diff --git a/src/query/ast/src/parser/expr.rs b/src/query/ast/src/parser/expr.rs index ad89dfc6dc17..0202ca03b3c5 100644 --- a/src/query/ast/src/parser/expr.rs +++ b/src/query/ast/src/parser/expr.rs @@ -1126,9 +1126,7 @@ pub fn type_name(i: Input) -> IResult { let ty_date = value(TypeName::Date, rule! { DATE }); let ty_datetime = map( rule! { (DATETIME | TIMESTAMP) ~ ( "(" ~ #literal_u64 ~ ")" )? }, - |(_, opt_precision)| TypeName::Timestamp { - precision: opt_precision.map(|(_, precision, _)| precision), - }, + |(_, _)| TypeName::Timestamp, ); let ty_string = value( TypeName::String, diff --git a/src/query/ast/tests/it/testdata/statement.txt b/src/query/ast/tests/it/testdata/statement.txt index d9a0f96e95e3..753cb93ed275 100644 --- a/src/query/ast/tests/it/testdata/statement.txt +++ b/src/query/ast/tests/it/testdata/statement.txt @@ -887,7 +887,7 @@ DropDatabase( ---------- Input ---------- create table c(a DateTime null, b DateTime(3)); ---------- Output --------- -CREATE TABLE c (a Timestamp NULL, b Timestamp(3) NOT NULL) +CREATE TABLE c (a TIMESTAMP NULL, b TIMESTAMP NOT NULL) ---------- AST ------------ CreateTable( CreateTableStmt { @@ -909,9 +909,7 @@ CreateTable( span: Ident(15..16), }, data_type: Nullable( - Timestamp { - precision: None, - }, + Timestamp, ), default_expr: None, comment: None, @@ -922,11 +920,7 @@ CreateTable( quote: None, span: Ident(32..33), }, - data_type: Timestamp { - precision: Some( - 3, - ), - }, + data_type: Timestamp, default_expr: None, comment: None, }, diff --git a/src/query/datablocks/tests/it/data_block.rs b/src/query/datablocks/tests/it/data_block.rs index d26cb7a7735c..297cfd56c012 100644 --- a/src/query/datablocks/tests/it/data_block.rs +++ b/src/query/datablocks/tests/it/data_block.rs @@ -45,8 +45,8 @@ fn test_data_block() -> Result<()> { fn test_data_block_convert() -> Result<()> { let schema = DataSchemaRefExt::create(vec![ DataField::new("a", DateType::new_impl()), - DataField::new("b", TimestampType::new_impl(0)), - DataField::new("b", TimestampType::new_impl(3)), + DataField::new("b", TimestampType::new_impl()), + DataField::new("b", TimestampType::new_impl()), ]); let block = DataBlock::create(schema.clone(), vec![ diff --git a/src/query/datavalues/src/types/data_type.rs b/src/query/datavalues/src/types/data_type.rs index 68241f262e90..c84f77829726 100644 --- a/src/query/datavalues/src/types/data_type.rs +++ b/src/query/datavalues/src/types/data_type.rs @@ -19,7 +19,6 @@ use std::fmt::Formatter; use common_arrow::arrow::datatypes::DataType as ArrowType; use common_arrow::arrow::datatypes::Field as ArrowField; -use common_arrow::arrow::datatypes::TimeUnit; use common_exception::Result; use dyn_clone::DynClone; use enum_dispatch::enum_dispatch; @@ -189,19 +188,7 @@ pub fn from_arrow_type(dt: &ArrowType) -> DataTypeImpl { DataTypeImpl::String(StringType::default()) } - ArrowType::Timestamp(TimeUnit::Second, _) => { - DataTypeImpl::Timestamp(TimestampType::create(0)) - } - ArrowType::Timestamp(TimeUnit::Millisecond, _) => { - DataTypeImpl::Timestamp(TimestampType::create(3)) - } - ArrowType::Timestamp(TimeUnit::Microsecond, _) => { - DataTypeImpl::Timestamp(TimestampType::create(6)) - } - // At most precision is 6 - ArrowType::Timestamp(TimeUnit::Nanosecond, _) => { - DataTypeImpl::Timestamp(TimestampType::create(6)) - } + ArrowType::Timestamp(_, _) => DataTypeImpl::Timestamp(TimestampType), ArrowType::Date32 | ArrowType::Date64 => DataTypeImpl::Date(DateType::default()), @@ -232,14 +219,7 @@ pub fn from_arrow_field(f: &ArrowField) -> DataTypeImpl { "Date" => return DateType::new_impl(), // OLD COMPATIBLE Behavior - "Timestamp" => match metadata { - Some(meta) => { - let mut chars = meta.chars(); - let precision = chars.next().unwrap().to_digit(10).unwrap(); - return TimestampType::new_impl(precision as usize); - } - None => return TimestampType::new_impl(0), - }, + "Timestamp" => return TimestampType::new_impl(), "Interval" => return IntervalType::new_impl(metadata.unwrap().into()), "Variant" => return VariantType::new_impl(), "VariantArray" => return VariantArrayType::new_impl(), diff --git a/src/query/datavalues/src/types/deserializations/timestamp.rs b/src/query/datavalues/src/types/deserializations/timestamp.rs index f96902bfcb1a..8bfbb2647b6a 100644 --- a/src/query/datavalues/src/types/deserializations/timestamp.rs +++ b/src/query/datavalues/src/types/deserializations/timestamp.rs @@ -21,7 +21,6 @@ use crate::prelude::*; pub struct TimestampDeserializer { pub builder: MutablePrimitiveColumn, - pub precision: usize, } impl TypeDeserializer for TimestampDeserializer { diff --git a/src/query/datavalues/src/types/eq.rs b/src/query/datavalues/src/types/eq.rs index 08f716e37e58..a4e13a2f7aea 100644 --- a/src/query/datavalues/src/types/eq.rs +++ b/src/query/datavalues/src/types/eq.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use super::type_array::ArrayType; use super::type_nullable::NullableType; use super::type_struct::StructType; -use super::type_timestamp::TimestampType; use super::DataType; use super::DataTypeImpl; @@ -49,16 +48,8 @@ pub fn equal(lhs: &DataTypeImpl, rhs: &DataTypeImpl) -> bool { use crate::prelude::TypeID::*; match lhs.data_type_id() { Boolean | UInt8 | UInt16 | UInt32 | UInt64 | Int8 | Int16 | Int32 | Int64 | Float32 - | Float64 | String | Date | Interval | Null | Variant | VariantArray | VariantObject => { - true - } - - Timestamp => { - let lhs: TimestampType = lhs.to_owned().try_into().unwrap(); - let rhs: TimestampType = rhs.to_owned().try_into().unwrap(); - - lhs.precision() == rhs.precision() - } + | Float64 | String | Date | Timestamp | Interval | Null | Variant | VariantArray + | VariantObject => true, Nullable => { let lhs: NullableType = lhs.to_owned().try_into().unwrap(); diff --git a/src/query/datavalues/src/types/serializations/timestamp.rs b/src/query/datavalues/src/types/serializations/timestamp.rs index e983a29b72c2..a72617be6738 100644 --- a/src/query/datavalues/src/types/serializations/timestamp.rs +++ b/src/query/datavalues/src/types/serializations/timestamp.rs @@ -28,26 +28,14 @@ const TIME_FMT: &str = "%Y-%m-%d %H:%M:%S"; #[derive(Debug, Clone)] pub struct TimestampSerializer<'a> { - #[allow(unused)] - pub(crate) precision: usize, pub(crate) values: &'a [i64], - pub(crate) date_format: String, } impl<'a> TimestampSerializer<'a> { - pub fn try_create(precision: usize, col: &'a ColumnRef) -> Result { + pub fn try_create(col: &'a ColumnRef) -> Result { let col: &PrimitiveColumn = Series::check_get(col)?; - let date_format = { - if precision == 0 { - "%Y-%m-%d %H:%M:%S".to_string() - } else { - format!("%Y-%m-%d %H:%M:%S%.{}f", precision) - } - }; Ok(Self { - precision, values: col.values(), - date_format, }) } pub fn to_timestamp(&self, value: &i64, tz: &Tz) -> DateTime { @@ -62,7 +50,7 @@ impl<'a> TypeSerializer<'a> for TimestampSerializer<'a> { fn write_field(&self, row_index: usize, buf: &mut Vec, format: &FormatSettings) { let dt = self.to_timestamp(&self.values[row_index], &format.timezone); - let s = dt.format(&self.date_format).to_string(); + let s = dt.format("%Y-%m-%d %H:%M:%S%.6f").to_string(); buf.extend_from_slice(s.as_bytes()) } diff --git a/src/query/datavalues/src/types/type_coercion.rs b/src/query/datavalues/src/types/type_coercion.rs index d98f27ced162..b706618b3c85 100644 --- a/src/query/datavalues/src/types/type_coercion.rs +++ b/src/query/datavalues/src/types/type_coercion.rs @@ -268,7 +268,7 @@ pub fn compare_coercion(lhs_type: &DataTypeImpl, rhs_type: &DataTypeImpl) -> Res { if (lhs_id.is_numeric() || lhs_id.is_string()) && rhs_id.is_date_or_date_time() { return if lhs_id.is_string() { - Ok(TimestampType::new_impl(6)) + Ok(TimestampType::new_impl()) } else { Ok(rhs_type.clone()) }; @@ -276,7 +276,7 @@ pub fn compare_coercion(lhs_type: &DataTypeImpl, rhs_type: &DataTypeImpl) -> Res if (rhs_id.is_numeric() || rhs_id.is_string()) && lhs_id.is_date_or_date_time() { return if rhs_id.is_string() { - Ok(TimestampType::new_impl(6)) + Ok(TimestampType::new_impl()) } else { Ok(lhs_type.clone()) }; @@ -287,12 +287,7 @@ pub fn compare_coercion(lhs_type: &DataTypeImpl, rhs_type: &DataTypeImpl) -> Res return match (lhs_id, rhs_id) { (TypeID::Date, _) => Ok(rhs_type.clone()), (_, TypeID::Date) => Ok(lhs_type.clone()), - (TypeID::Timestamp, TypeID::Timestamp) => { - let lhs: TimestampType = lhs_type.to_owned().try_into()?; - let rhs: TimestampType = rhs_type.to_owned().try_into()?; - let precision = cmp::max(lhs.precision(), rhs.precision()); - Ok(TimestampType::new_impl(precision)) - } + (TypeID::Timestamp, TypeID::Timestamp) => Ok(lhs_type.clone()), _ => unreachable!(), }; } diff --git a/src/query/datavalues/src/types/type_factory.rs b/src/query/datavalues/src/types/type_factory.rs index 374d786ac485..8d27d7f19693 100644 --- a/src/query/datavalues/src/types/type_factory.rs +++ b/src/query/datavalues/src/types/type_factory.rs @@ -54,11 +54,7 @@ static TYPE_FACTORY: Lazy> = Lazy::new(|| { type_factory.register(VariantObjectType::new_impl()); // Timestamp is a special case - { - for precision in 0..7 { - type_factory.register(TimestampType::new_impl(precision)); - } - } + type_factory.register(TimestampType::new_impl()); Arc::new(type_factory) }); diff --git a/src/query/datavalues/src/types/type_timestamp.rs b/src/query/datavalues/src/types/type_timestamp.rs index b33ff87645d5..c13a642f4cc9 100644 --- a/src/query/datavalues/src/types/type_timestamp.rs +++ b/src/query/datavalues/src/types/type_timestamp.rs @@ -48,22 +48,11 @@ pub fn check_timestamp(micros: i64) -> Result<()> { /// Timestamp type only stores UTC time in microseconds #[derive(Default, Clone, Hash, serde::Deserialize, serde::Serialize)] -pub struct TimestampType { - /// Typically are used - 0 (seconds) 3 (milliseconds), 6 (microseconds) - precision: usize, -} +pub struct TimestampType; impl TimestampType { - pub fn create(precision: usize) -> Self { - TimestampType { precision } - } - - pub fn new_impl(precision: usize) -> DataTypeImpl { - DataTypeImpl::Timestamp(TimestampType { precision }) - } - - pub fn precision(&self) -> usize { - self.precision + pub fn new_impl() -> DataTypeImpl { + DataTypeImpl::Timestamp(TimestampType) } #[inline] @@ -76,12 +65,8 @@ impl TimestampType { v / 1_000_000 } - pub fn format_string(&self) -> String { - if self.precision == 0 { - "%Y-%m-%d %H:%M:%S".to_string() - } else { - format!("%Y-%m-%d %H:%M:%S%.{}f", self.precision) - } + pub fn format_string(&self) -> &str { + "%Y-%m-%d %H:%M:%S%.6f" } } @@ -96,20 +81,11 @@ impl DataType for TimestampType { } fn name(&self) -> String { - format!("Timestamp({})", self.precision) + format!("Timestamp") } fn aliases(&self) -> &[&str] { - match self.precision { - 0 => &["DateTime(0)"], - 1 => &["DateTime(1)"], - 2 => &["DateTime(2)"], - 3 => &["DateTime(3)"], - 4 => &["DateTime(4)"], - 5 => &["DateTime(5)"], - 6 => &["Timestamp", "DateTime", "DateTime(6)"], - _ => &[], - } + &["DateTime"] } fn default_value(&self) -> DataValue { @@ -143,13 +119,12 @@ impl DataType for TimestampType { } fn create_serializer_inner<'a>(&self, col: &'a ColumnRef) -> Result> { - Ok(TimestampSerializer::<'a>::try_create(self.precision, col)?.into()) + Ok(TimestampSerializer::<'a>::try_create(col)?.into()) } fn create_deserializer(&self, capacity: usize) -> TypeDeserializerImpl { TimestampDeserializer { builder: MutablePrimitiveColumn::::with_capacity(capacity), - precision: self.precision, } .into() } @@ -161,6 +136,6 @@ impl DataType for TimestampType { impl std::fmt::Debug for TimestampType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Timestamp({})", self.precision()) + write!(f, "Timestamp") } } diff --git a/src/query/datavalues/src/types/type_traits.rs b/src/query/datavalues/src/types/type_traits.rs index ace3abb4d9b0..d25c2334ba59 100644 --- a/src/query/datavalues/src/types/type_traits.rs +++ b/src/query/datavalues/src/types/type_traits.rs @@ -117,7 +117,7 @@ impl ToDateType for i32 { impl ToDateType for i64 { fn to_date_type() -> DataTypeImpl { - TimestampType::new_impl(6) + TimestampType::new_impl() } } diff --git a/src/query/datavalues/tests/it/types/create_column.rs b/src/query/datavalues/tests/it/types/create_column.rs index 47e0150f0f25..2a2e2bdbd033 100644 --- a/src/query/datavalues/tests/it/types/create_column.rs +++ b/src/query/datavalues/tests/it/types/create_column.rs @@ -45,14 +45,14 @@ fn test_create_constant() -> Result<()> { }, Test { name: "datetime32", - data_type: TimestampType::new_impl(0), + data_type: TimestampType::new_impl(), value: DataValue::UInt64(1630320462), size: 2, column_expected: Series::from_data(vec![1630320462i64, 1630320462]), }, Test { name: "datetime64", - data_type: TimestampType::new_impl(3), + data_type: TimestampType::new_impl(), value: DataValue::Int64(1630320462), size: 2, column_expected: Series::from_data(vec![1630320462i64, 1630320462]), diff --git a/src/query/datavalues/tests/it/types/serializations/mod.rs b/src/query/datavalues/tests/it/types/serializations/mod.rs index bd4bf0f1a50b..a2815a0d9786 100644 --- a/src/query/datavalues/tests/it/types/serializations/mod.rs +++ b/src/query/datavalues/tests/it/types/serializations/mod.rs @@ -50,7 +50,7 @@ fn test_serializers() -> Result<()> { }, Test { name: "datetime32", - data_type: TimestampType::new_impl(0), + data_type: TimestampType::new_impl(), column: Series::from_data(vec![1630320462000000i64, 1637117572000000i64, 1000000]), val_str: "2021-08-30 10:47:42", col_str: vec![ @@ -184,7 +184,7 @@ fn test_serializers() -> Result<()> { #[test] fn test_convert_arrow() { - let t = TimestampType::new_impl(6); + let t = TimestampType::new_impl(); let arrow_y = t.to_arrow_field("x"); let new_t = from_arrow_field(&arrow_y); diff --git a/src/query/expression/src/converts/from.rs b/src/query/expression/src/converts/from.rs index d45749bda67a..3c82c4ee1e3d 100644 --- a/src/query/expression/src/converts/from.rs +++ b/src/query/expression/src/converts/from.rs @@ -101,9 +101,9 @@ pub fn from_scalar(datavalue: &DataValue, datatype: &DataTypeImpl) -> Scalar { DataTypeImpl::Float64(_) => { Scalar::Number(NumberScalar::Float64(datavalue.as_f64().unwrap().into())) } - DataTypeImpl::Timestamp(t) => Scalar::Timestamp(Timestamp { + DataTypeImpl::Timestamp(_) => Scalar::Timestamp(Timestamp { ts: datavalue.as_i64().unwrap(), - precision: t.precision() as u8, + precision: 6, }), DataTypeImpl::Date(_) => Scalar::Date(datavalue.as_i64().unwrap() as i32), DataTypeImpl::Interval(_) => Scalar::Interval(datavalue.as_i64().unwrap() as i64), diff --git a/src/query/expression/tests/it/convert.rs b/src/query/expression/tests/it/convert.rs index 65632af7c7dd..700b0a7c99b9 100644 --- a/src/query/expression/tests/it/convert.rs +++ b/src/query/expression/tests/it/convert.rs @@ -41,7 +41,7 @@ fn random_type() -> DataTypeImpl { 12 => f64::to_data_type(), 13 => DataTypeImpl::Date(DateType {}), - 14 => DataTypeImpl::Timestamp(TimestampType::create(3)), + 14 => DataTypeImpl::Timestamp(TimestampType::create()), 15 => DataTypeImpl::String(StringType {}), 16 => DataTypeImpl::Struct(StructType::create(None, vec![ f32::to_data_type(), diff --git a/src/query/functions/src/scalars/dates/interval_function.rs b/src/query/functions/src/scalars/dates/interval_function.rs index 92d6e20e48cc..e30c6989bf54 100644 --- a/src/query/functions/src/scalars/dates/interval_function.rs +++ b/src/query/functions/src/scalars/dates/interval_function.rs @@ -58,18 +58,15 @@ where T: IntervalArithmeticImpl + Send + Sync + Clone + 'static display_name, T::DateResultType::to_date_type(), T::eval_date, - factor, - 0, + factor ), TypeID::Timestamp => { let ts: TimestampType = args[0].to_owned().try_into()?; - let precision = ts.precision(); IntervalFunction::::try_create_func( display_name, args[0].clone(), T::eval_timestamp, - factor, - precision, + factor ) }, _=> Err(ErrorCode::BadDataValueType(format!( @@ -95,7 +92,6 @@ pub struct IntervalFunction, } @@ -111,14 +107,12 @@ where result_type: DataTypeImpl, func: F, factor: i64, - precision: usize, ) -> Result> { Ok(Box::new(Self { display_name: display_name.to_string(), result_type, func, factor, - precision, _phantom: PhantomData, })) } @@ -146,7 +140,7 @@ where _input_rows: usize, ) -> Result { // Todo(zhyass): define the ctx out of the eval. - let mut ctx = EvalContext::new(self.factor, self.precision, None, func_ctx.tz); + let mut ctx = EvalContext::new(self.factor, None, func_ctx.tz); let col = scalar_binary_op( columns[0].column(), columns[1].column(), @@ -211,7 +205,7 @@ impl IntervalArithmeticImpl for AddDaysImpl { } fn eval_timestamp(l: i64, r: impl AsPrimitive, ctx: &mut EvalContext) -> i64 { - let base = 10_i64.pow(ctx.precision as u32); + let base = 10_i64.pow(6); let factor = ctx.factor * 24 * 3600 * base; (l as i64).wrapping_add(r.as_() * factor) } @@ -229,7 +223,7 @@ impl IntervalArithmeticImpl for AddTimesImpl { } fn eval_timestamp(l: i64, r: impl AsPrimitive, ctx: &mut EvalContext) -> i64 { - let base = 10_i64.pow(ctx.precision as u32); + let base = 10_i64.pow(6); let factor = ctx.factor * base; (l as i64).wrapping_add(r.as_() * factor) } diff --git a/src/query/functions/src/scalars/dates/now.rs b/src/query/functions/src/scalars/dates/now.rs index 4fa37cdc7da6..a0186047a8b1 100644 --- a/src/query/functions/src/scalars/dates/now.rs +++ b/src/query/functions/src/scalars/dates/now.rs @@ -49,7 +49,7 @@ impl Function for NowFunction { } fn return_type(&self) -> DataTypeImpl { - TimestampType::new_impl(6) + TimestampType::new_impl() } fn eval( diff --git a/src/query/functions/src/scalars/dates/number_function.rs b/src/query/functions/src/scalars/dates/number_function.rs index 8a9c07c022b7..68ab2528f20b 100644 --- a/src/query/functions/src/scalars/dates/number_function.rs +++ b/src/query/functions/src/scalars/dates/number_function.rs @@ -269,7 +269,7 @@ impl NumberOperator for ToMinute { // ToMinute is NOT a monotonic function in general, unless the time range is within the same hour. fn factor_function(_input_type: DataTypeImpl) -> Option> { Some( - RoundFunction::try_create("toStartOfHour", &[&TimestampType::new_impl(0)], Round::Hour) + RoundFunction::try_create("toStartOfHour", &[&TimestampType::new_impl()], Round::Hour) .unwrap(), ) } @@ -290,7 +290,7 @@ impl NumberOperator for ToSecond { Some( RoundFunction::try_create( "to_start_of_minute", - &[&TimestampType::new_impl(0)], + &[&TimestampType::new_impl()], Round::Minute, ) .unwrap(), diff --git a/src/query/functions/src/scalars/dates/round_function.rs b/src/query/functions/src/scalars/dates/round_function.rs index 7b616ca0086e..5d92dcd9ff71 100644 --- a/src/query/functions/src/scalars/dates/round_function.rs +++ b/src/query/functions/src/scalars/dates/round_function.rs @@ -114,7 +114,7 @@ impl Function for RoundFunction { } fn return_type(&self) -> DataTypeImpl { - TimestampType::new_impl(0) + TimestampType::new_impl() } fn eval( diff --git a/src/query/functions/src/scalars/expressions/cast_from_datetimes.rs b/src/query/functions/src/scalars/expressions/cast_from_datetimes.rs index 2a866f66abcf..9e89c6796944 100644 --- a/src/query/functions/src/scalars/expressions/cast_from_datetimes.rs +++ b/src/query/functions/src/scalars/expressions/cast_from_datetimes.rs @@ -87,7 +87,7 @@ pub fn cast_from_timestamp( for v in c.iter() { let s = timestamp_to_string( DateConverter::to_timestamp(v, &tz), - date_time64.format_string().as_str(), + date_time64.format_string(), ); builder.append_value(s.as_bytes()); } diff --git a/src/query/functions/src/scalars/expressions/ctx.rs b/src/query/functions/src/scalars/expressions/ctx.rs index 10f5cca1c98c..4884c58755b6 100644 --- a/src/query/functions/src/scalars/expressions/ctx.rs +++ b/src/query/functions/src/scalars/expressions/ctx.rs @@ -18,7 +18,6 @@ use common_exception::ErrorCode; #[derive(Debug, Clone)] pub struct EvalContext { pub factor: i64, - pub precision: usize, pub error: Option, pub tz: Tz, } @@ -28,7 +27,6 @@ impl Default for EvalContext { let tz = "UTC".parse::().unwrap(); Self { factor: 1, - precision: 0, error: None, tz, } @@ -36,13 +34,8 @@ impl Default for EvalContext { } impl EvalContext { - pub fn new(factor: i64, precision: usize, error: Option, tz: Tz) -> Self { - Self { - factor, - precision, - error, - tz, - } + pub fn new(factor: i64, error: Option, tz: Tz) -> Self { + Self { factor, error, tz } } pub fn set_error(&mut self, e: ErrorCode) { diff --git a/src/query/functions/tests/it/aggregates/aggregate_function.rs b/src/query/functions/tests/it/aggregates/aggregate_function.rs index 79f70dcf719a..3e277f368ad1 100644 --- a/src/query/functions/tests/it/aggregates/aggregate_function.rs +++ b/src/query/functions/tests/it/aggregates/aggregate_function.rs @@ -52,7 +52,7 @@ fn test_aggregate_function() -> Result<()> { DataField::new("a", i64::to_data_type()), DataField::new("b", i64::to_data_type()), // args for window funnel function - DataField::new("dt", TimestampType::new_impl(0)), + DataField::new("dt", TimestampType::new_impl()), DataField::new("event = 1001", bool::to_data_type()), DataField::new("event = 1002", bool::to_data_type()), DataField::new("event = 1003", bool::to_data_type()), @@ -348,7 +348,7 @@ fn test_aggregate_function_with_group_by() -> Result<()> { DataField::new("b", i64::to_data_type()), DataField::new("c", Vu8::to_data_type()), // args for window funnel function - DataField::new("dt", TimestampType::new_impl(0)), + DataField::new("dt", TimestampType::new_impl()), DataField::new("event = 1001", bool::to_data_type()), DataField::new("event = 1002", bool::to_data_type()), DataField::new("event = 1003", bool::to_data_type()), diff --git a/src/query/functions/tests/it/scalars/arithmetics.rs b/src/query/functions/tests/it/scalars/arithmetics.rs index 651e2ddbce37..04e14be54d2f 100644 --- a/src/query/functions/tests/it/scalars/arithmetics.rs +++ b/src/query/functions/tests/it/scalars/arithmetics.rs @@ -160,7 +160,7 @@ fn test_arithmetic_date_interval() -> Result<()> { to_microseconds(2020, 2, 29, 10, 30, 00, 00), // 2020-2-29 10:30:00 to_microseconds(2021, 2, 28, 10, 30, 00, 00), // 2021-2-28 10:30:00 ]), - DataField::new("dummy_0", TimestampType::new_impl(0)), + DataField::new("dummy_0", TimestampType::new_impl()), ), ColumnWithField::new( Series::from_data(vec![1i64, -1]), @@ -181,7 +181,7 @@ fn test_arithmetic_date_interval() -> Result<()> { to_microseconds(2020, 2, 29, 10, 30, 00, 000), // 2020-2-29 10:30:00.000 to_microseconds(1960, 2, 29, 10, 30, 00, 000), // 1960-2-29 10:30:00.000 ]), - DataField::new("dummy_0", TimestampType::new_impl(3)), + DataField::new("dummy_0", TimestampType::new_impl()), ), ColumnWithField::new( Series::from_data(vec![1i64, -4]), @@ -223,7 +223,7 @@ fn test_arithmetic_date_interval() -> Result<()> { to_microseconds(2020, 3, 31, 10, 30, 00, 00), // 2020-3-31 10:30:00 to_microseconds(2000, 1, 31, 10, 30, 00, 00), // 2000-1-31 10:30:00 ]), - DataField::new("dummy_0", TimestampType::new_impl(0)), + DataField::new("dummy_0", TimestampType::new_impl()), ), ColumnWithField::new( Series::from_data(vec![-1i64, 241]), @@ -265,7 +265,7 @@ fn test_arithmetic_date_interval() -> Result<()> { to_microseconds(2020, 2, 29, 10, 30, 00, 000), // 2020-2-29 10:30:00.000 to_microseconds(1960, 2, 29, 10, 30, 00, 000), // 1960-2-29 10:30:00.000 ]), - DataField::new("dummy_0", TimestampType::new_impl(6)), + DataField::new("dummy_0", TimestampType::new_impl()), ), ColumnWithField::new( Series::from_data(vec![-30i64, 30]), @@ -328,7 +328,7 @@ fn test_arithmetic_date_interval() -> Result<()> { to_microseconds(2020, 3, 31, 10, 30, 00, 00), // 2020-3-31 10:30:00 to_microseconds(2000, 1, 31, 10, 30, 00, 00), // 2000-1-31 10:30:00 ]), - DataField::new("dummy_0", TimestampType::new_impl(6)), + DataField::new("dummy_0", TimestampType::new_impl()), ), ColumnWithField::new( Series::from_data(vec![-120i64, 23]), diff --git a/src/query/functions/tests/it/scalars/dates/date.rs b/src/query/functions/tests/it/scalars/dates/date.rs index 18d8026e82a7..b2d184602f65 100644 --- a/src/query/functions/tests/it/scalars/dates/date.rs +++ b/src/query/functions/tests/it/scalars/dates/date.rs @@ -37,7 +37,7 @@ fn test_round_function() -> Result<()> { name: "test-timeSlot-now", columns: vec![ColumnWithField::new( Series::from_data(vec![1630812366000000i64, 1630839682000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![ 1630812366i64 / r * r * MICROSECONDS, @@ -56,7 +56,7 @@ fn test_to_start_of_function() -> Result<()> { name: "test-timeSlot-now", columns: vec![ColumnWithField::new( Series::from_data(vec![1631705259000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![18809i32]), error: "", diff --git a/src/query/functions/tests/it/scalars/dates/date_function.rs b/src/query/functions/tests/it/scalars/dates/date_function.rs index 0663b63e4a2f..ea5d0462556f 100644 --- a/src/query/functions/tests/it/scalars/dates/date_function.rs +++ b/src/query/functions/tests/it/scalars/dates/date_function.rs @@ -46,7 +46,7 @@ fn test_to_yyyymm_function() -> Result<()> { name: "test_to_yyyymm_datetime", columns: vec![ColumnWithField::new( Series::from_data(vec![0i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![197001u32]), error: "", @@ -73,7 +73,7 @@ fn test_to_yyyymm_function() -> Result<()> { name: "test_to_yyyymm_constant_datetime", columns: vec![ColumnWithField::new( Arc::new(ConstColumn::new(Series::from_data(vec![0i64]), 1)), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![197001u32]), error: "", @@ -108,7 +108,7 @@ fn test_to_yyyymmdd_function() -> Result<()> { name: "test_to_yyyymmdd_datetime", columns: vec![ColumnWithField::new( Series::from_data(vec![1630833797000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![20210905u32]), error: "", @@ -135,7 +135,7 @@ fn test_to_yyyymmdd_function() -> Result<()> { name: "test_to_yyyymmdd_datetime", columns: vec![ColumnWithField::new( Series::from_data(vec![1630833797000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![20210905u32]), error: "", @@ -165,7 +165,7 @@ fn test_to_yyyymmdd_function() -> Result<()> { Series::from_data(vec![1630833797000000i64]), 1, )), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![20210905u32]), error: "", @@ -200,7 +200,7 @@ fn test_to_yyyymmddhhmmss_function() -> Result<()> { name: "test_to_yyyymmddhhmmss_datetime", columns: vec![ColumnWithField::new( Series::from_data(vec![1630833797000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![20210905092317u64]), error: "", @@ -230,7 +230,7 @@ fn test_to_yyyymmddhhmmss_function() -> Result<()> { Series::from_data(vec![1630833797000000i64]), 1, )), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![20210905092317u64]), error: "", @@ -265,7 +265,7 @@ fn test_to_month_function() -> Result<()> { name: "test_to_month_datetime", columns: vec![ColumnWithField::new( Series::from_data(vec![1633081817000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![10u8]), error: "", @@ -295,7 +295,7 @@ fn test_to_month_function() -> Result<()> { Series::from_data(vec![1633081817000000i64]), 1, )), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![10u8]), error: "", @@ -330,7 +330,7 @@ fn test_to_day_of_year_function() -> Result<()> { name: "test_to_day_of_year_datetime", columns: vec![ColumnWithField::new( Series::from_data(vec![1633173324000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![275u16]), error: "", @@ -360,7 +360,7 @@ fn test_to_day_of_year_function() -> Result<()> { Series::from_data(vec![1633173324000000i64]), 1, )), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![275u16]), error: "", @@ -395,7 +395,7 @@ fn test_todatefweek_function() -> Result<()> { name: "test_to_day_of_week_datetime", columns: vec![ColumnWithField::new( Series::from_data(vec![1633173324000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![6u8]), error: "", @@ -425,7 +425,7 @@ fn test_todatefweek_function() -> Result<()> { Series::from_data(vec![1633173324000000i64]), 1, )), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![6u8]), error: "", @@ -460,7 +460,7 @@ fn test_to_day_of_month_function() -> Result<()> { name: "test_to_day_of_month_datetime", columns: vec![ColumnWithField::new( Series::from_data(vec![1633173324000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![2u8]), error: "", @@ -490,7 +490,7 @@ fn test_to_day_of_month_function() -> Result<()> { Series::from_data(vec![1633173324000000i64]), 1, )), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![2u8]), error: "", @@ -525,7 +525,7 @@ fn test_to_hour_function() -> Result<()> { name: "test_to_hour_datetime", columns: vec![ColumnWithField::new( Series::from_data(vec![1634551542000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![10u8]), error: "", @@ -555,7 +555,7 @@ fn test_to_hour_function() -> Result<()> { Series::from_data(vec![1634551542000000i64]), 1, )), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![10u8]), error: "", @@ -590,7 +590,7 @@ fn test_to_minute_function() -> Result<()> { name: "test_to_minute_datetime", columns: vec![ColumnWithField::new( Series::from_data(vec![1634551542000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![5u8]), error: "", @@ -620,7 +620,7 @@ fn test_to_minute_function() -> Result<()> { Series::from_data(vec![1634551542000000i64]), 1, )), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![5u8]), error: "", @@ -655,7 +655,7 @@ fn test_to_second_function() -> Result<()> { name: "test_to_second_datetime", columns: vec![ColumnWithField::new( Series::from_data(vec![1634551542000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![42u8]), error: "", @@ -685,7 +685,7 @@ fn test_to_second_function() -> Result<()> { Series::from_data(vec![1634551542000000i64]), 1, )), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![42u8]), error: "", @@ -720,7 +720,7 @@ fn test_to_monday_function() -> Result<()> { name: "test_to_monday_datetime", columns: vec![ColumnWithField::new( Series::from_data(vec![1634614318000000i64]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec![18918u16]), error: "", diff --git a/src/query/functions/tests/it/scalars/dates/interval_function.rs b/src/query/functions/tests/it/scalars/dates/interval_function.rs index e07d86f12a61..ae1b744ab26a 100644 --- a/src/query/functions/tests/it/scalars/dates/interval_function.rs +++ b/src/query/functions/tests/it/scalars/dates/interval_function.rs @@ -31,7 +31,7 @@ fn test_add_months() -> Result<()> { let schema = DataSchemaRefExt::create(vec![ DataField::new("date", DateType::new_impl()), - DataField::new("datetime", TimestampType::new_impl(0)), + DataField::new("datetime", TimestampType::new_impl()), DataField::new("u8", u8::to_data_type()), DataField::new("u16", u16::to_data_type()), DataField::new("u32", u32::to_data_type()), @@ -116,7 +116,7 @@ fn test_add_months() -> Result<()> { expects.reserve(10); for (field, arg) in fields.iter().zip(args.iter()) { let add_months = AddMonthsFunction::try_create_func("add_months", 1, &[ - &TimestampType::new_impl(0), + &TimestampType::new_impl(), arg, ])?; let col = add_months.eval( @@ -150,7 +150,7 @@ fn test_add_subtract_seconds() -> Result<()> { let dt_to_seconds = |dt: &str| -> i64 { DateTime::parse_from_rfc3339(dt).unwrap().timestamp() }; let schema = DataSchemaRefExt::create(vec![ - DataField::new("datetime", TimestampType::new_impl(0)), + DataField::new("datetime", TimestampType::new_impl()), DataField::new("u8", u8::to_data_type()), DataField::new("u16", u16::to_data_type()), DataField::new("u32", u32::to_data_type()), @@ -205,7 +205,7 @@ fn test_add_subtract_seconds() -> Result<()> { expects.reserve(10); for (field, arg) in fields.iter().zip(args.iter()) { let add_seconds = AddTimesFunction::try_create_func("add_seconds", 1, &[ - &TimestampType::new_impl(0), + &TimestampType::new_impl(), arg, ])?; let col = add_seconds.eval( @@ -236,7 +236,7 @@ fn test_add_subtract_seconds() -> Result<()> { expects.reserve(10); for (field, arg) in fields.iter().zip(args.iter()) { let add_seconds = AddTimesFunction::try_create_func("subtract_seconds", -1, &[ - &TimestampType::new_impl(0), + &TimestampType::new_impl(), arg, ])?; let col = add_seconds.eval( diff --git a/src/query/functions/tests/it/scalars/expressions.rs b/src/query/functions/tests/it/scalars/expressions.rs index 81d931eed71c..fab446ffa7f0 100644 --- a/src/query/functions/tests/it/scalars/expressions.rs +++ b/src/query/functions/tests/it/scalars/expressions.rs @@ -123,7 +123,7 @@ fn test_datetime_cast_function() -> Result<()> { name: "cast-datetime-to-string-passed", columns: vec![ColumnWithField::new( Series::from_data(vec![1614906061000000i64, 1635070210000000]), - DataField::new("dummy_1", TimestampType::new_impl(0)), + DataField::new("dummy_1", TimestampType::new_impl()), )], expect: Series::from_data(vec!["2021-03-05 01:01:01", "2021-10-24 10:10:10"]), error: "", @@ -567,7 +567,7 @@ fn test_variant_cast_function() -> Result<()> { VariantValue::from(json!("2021-10-24 10:10:10")), ])], expect: Series::from_data(vec![1614906061000000i64, 1635070210000000]), - error: "Cast error happens in casting from Variant to Timestamp(6)", + error: "Cast error happens in casting from Variant to Timestamp", }), ]; diff --git a/src/query/functions/tests/it/scalars/others/running_difference.rs b/src/query/functions/tests/it/scalars/others/running_difference.rs index 510bd789c104..5461453df195 100644 --- a/src/query/functions/tests/it/scalars/others/running_difference.rs +++ b/src/query/functions/tests/it/scalars/others/running_difference.rs @@ -227,10 +227,7 @@ fn test_running_difference_datetime32_first_null() -> Result<()> { name: "datetime32_first_null", columns: vec![ColumnWithField::new( Series::from_data([None, Some(3_i64), None, Some(4), Some(10)]), - DataField::new( - "dummy_1", - NullableType::new_impl(TimestampType::new_impl(0)), - ), + DataField::new("dummy_1", NullableType::new_impl(TimestampType::new_impl())), )], expect: Series::from_data([None, None, None, None, Some(6_i64)]), error: "", @@ -239,10 +236,7 @@ fn test_running_difference_datetime32_first_null() -> Result<()> { name: "datetime32_first_not_null", columns: vec![ColumnWithField::new( Series::from_data([Some(2_i64), Some(3), None, Some(4), Some(10)]), - DataField::new( - "dummy_1", - NullableType::new_impl(TimestampType::new_impl(0)), - ), + DataField::new("dummy_1", NullableType::new_impl(TimestampType::new_impl())), )], expect: Series::from_data([Some(0_i64), Some(1), None, None, Some(6)]), error: "", diff --git a/src/query/legacy-expression/tests/it/monotonicity.rs b/src/query/legacy-expression/tests/it/monotonicity.rs index 881ab17177ec..679fc30b1879 100644 --- a/src/query/legacy-expression/tests/it/monotonicity.rs +++ b/src/query/legacy-expression/tests/it/monotonicity.rs @@ -48,7 +48,7 @@ fn create_u8(d: u8) -> Option { } fn create_datetime(d: i64) -> Option { - let data_field = DataField::new("x", TimestampType::new_impl(0)); + let data_field = DataField::new("x", TimestampType::new_impl()); let col = data_field .data_type() .create_constant_column(&DataValue::Int64(d), 1) @@ -61,7 +61,7 @@ fn verify_test(t: Test) -> Result<()> { let schema = DataSchemaRefExt::create(vec![ DataField::new("x", f64::to_data_type()), DataField::new("y", i64::to_data_type()), - DataField::new("z", TimestampType::new_impl(0)), + DataField::new("z", TimestampType::new_impl()), ]); let mut variables = HashMap::new(); diff --git a/src/query/legacy-parser/src/sql_common.rs b/src/query/legacy-parser/src/sql_common.rs index be74596143ff..2ec020bd6728 100644 --- a/src/query/legacy-parser/src/sql_common.rs +++ b/src/query/legacy-parser/src/sql_common.rs @@ -45,29 +45,8 @@ impl SQLCommon { SQLDataType::Boolean => Ok(bool::to_data_type()), SQLDataType::Date => Ok(DateType::new_impl()), // default precision is 6, microseconds - SQLDataType::Timestamp(None) | SQLDataType::DateTime(None) => { - Ok(TimestampType::new_impl(6)) - } - SQLDataType::Timestamp(Some(precision)) => { - if *precision <= 6 { - Ok(TimestampType::new_impl(*precision as usize)) - } else { - Err(ErrorCode::IllegalDataType(format!( - "The SQL data type TIMESTAMP(n), n only ranges from 0~6, {} is invalid", - precision - ))) - } - } - SQLDataType::DateTime(Some(precision)) => { - if *precision <= 6 { - Ok(TimestampType::new_impl(*precision as usize)) - } else { - Err(ErrorCode::IllegalDataType(format!( - "The SQL data type DATETIME(n), n only ranges from 0~6, {} is invalid", - precision - ))) - } - } + SQLDataType::Timestamp(_) | SQLDataType::DateTime(_) => Ok(TimestampType::new_impl()), + SQLDataType::Array(sql_type, nullable) => { let inner_data_type = Self::make_data_type(sql_type)?; if *nullable { diff --git a/src/query/service/src/sql/planner/binder/table.rs b/src/query/service/src/sql/planner/binder/table.rs index 63f831c5d7c7..6ecc5c09aec7 100644 --- a/src/query/service/src/sql/planner/binder/table.rs +++ b/src/query/service/src/sql/planner/binder/table.rs @@ -358,7 +358,7 @@ impl<'a> Binder { &[], ); let box (scalar, data_type) = type_checker - .resolve(expr, Some(TimestampType::new_impl(6))) + .resolve(expr, Some(TimestampType::new_impl())) .await?; if let Scalar::ConstantExpr(ConstantExpr { value, .. }) = scalar { diff --git a/src/query/service/src/sql/planner/semantic/type_check.rs b/src/query/service/src/sql/planner/semantic/type_check.rs index e57339123f24..85ad1898d3a4 100644 --- a/src/query/service/src/sql/planner/semantic/type_check.rs +++ b/src/query/service/src/sql/planner/semantic/type_check.rs @@ -1095,15 +1095,15 @@ impl<'a> TypeChecker<'a> { ) -> Result> { match interval_kind { IntervalKind::Year => { - self.resolve_function(span, "to_year", &[arg], Some(TimestampType::new_impl(0))) + self.resolve_function(span, "to_year", &[arg], Some(TimestampType::new_impl())) .await } IntervalKind::Quarter => { - self.resolve_function(span, "to_quarter", &[arg], Some(TimestampType::new_impl(0))) + self.resolve_function(span, "to_quarter", &[arg], Some(TimestampType::new_impl())) .await } IntervalKind::Month => { - self.resolve_function(span, "to_month", &[arg], Some(TimestampType::new_impl(0))) + self.resolve_function(span, "to_month", &[arg], Some(TimestampType::new_impl())) .await } IntervalKind::Day => { @@ -1111,20 +1111,20 @@ impl<'a> TypeChecker<'a> { span, "to_day_of_month", &[arg], - Some(TimestampType::new_impl(0)), + Some(TimestampType::new_impl()), ) .await } IntervalKind::Hour => { - self.resolve_function(span, "to_hour", &[arg], Some(TimestampType::new_impl(0))) + self.resolve_function(span, "to_hour", &[arg], Some(TimestampType::new_impl())) .await } IntervalKind::Minute => { - self.resolve_function(span, "to_minute", &[arg], Some(TimestampType::new_impl(0))) + self.resolve_function(span, "to_minute", &[arg], Some(TimestampType::new_impl())) .await } IntervalKind::Second => { - self.resolve_function(span, "to_second", &[arg], Some(TimestampType::new_impl(0))) + self.resolve_function(span, "to_second", &[arg], Some(TimestampType::new_impl())) .await } IntervalKind::Doy => { @@ -1132,7 +1132,7 @@ impl<'a> TypeChecker<'a> { span, "to_day_of_year", &[arg], - Some(TimestampType::new_impl(0)), + Some(TimestampType::new_impl()), ) .await } @@ -1141,7 +1141,7 @@ impl<'a> TypeChecker<'a> { span, "to_day_of_week", &[arg], - Some(TimestampType::new_impl(0)), + Some(TimestampType::new_impl()), ) .await } @@ -1294,7 +1294,7 @@ impl<'a> TypeChecker<'a> { span, "to_start_of_year", &[date], - Some(TimestampType::new_impl(0)), + Some(TimestampType::new_impl()), ) .await } @@ -1303,7 +1303,7 @@ impl<'a> TypeChecker<'a> { span, "to_start_of_quarter", &[date], - Some(TimestampType::new_impl(0)), + Some(TimestampType::new_impl()), ) .await } @@ -1312,7 +1312,7 @@ impl<'a> TypeChecker<'a> { span, "to_start_of_month", &[date], - Some(TimestampType::new_impl(0)), + Some(TimestampType::new_impl()), ) .await } @@ -1321,7 +1321,7 @@ impl<'a> TypeChecker<'a> { span, "to_start_of_day", &[date], - Some(TimestampType::new_impl(0)), + Some(TimestampType::new_impl()), ) .await } @@ -1330,7 +1330,7 @@ impl<'a> TypeChecker<'a> { span, "to_start_of_hour", &[date], - Some(TimestampType::new_impl(0)), + Some(TimestampType::new_impl()), ) .await } @@ -1339,7 +1339,7 @@ impl<'a> TypeChecker<'a> { span, "to_start_of_minute", &[date], - Some(TimestampType::new_impl(0)), + Some(TimestampType::new_impl()), ) .await } @@ -1348,7 +1348,7 @@ impl<'a> TypeChecker<'a> { span, "to_start_of_second", &[date], - Some(TimestampType::new_impl(0)), + Some(TimestampType::new_impl()), ) .await } diff --git a/src/query/storages/fuse/src/table_functions/fuse_blocks/fuse_block.rs b/src/query/storages/fuse/src/table_functions/fuse_blocks/fuse_block.rs index a7a4083bb8e0..5d69d1014e69 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_blocks/fuse_block.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_blocks/fuse_block.rs @@ -118,7 +118,7 @@ impl<'a> FuseBlock<'a> { pub fn schema() -> Arc { DataSchemaRefExt::create(vec![ DataField::new("snapshot_id", Vu8::to_data_type()), - DataField::new_nullable("timestamp", TimestampType::new_impl(6)), + DataField::new_nullable("timestamp", TimestampType::new_impl()), DataField::new("block_location", Vu8::to_data_type()), DataField::new("block_size", u64::to_data_type()), DataField::new_nullable("bloom_filter_location", Vu8::to_data_type()), diff --git a/src/query/storages/fuse/src/table_functions/fuse_snapshots/fuse_snapshot.rs b/src/query/storages/fuse/src/table_functions/fuse_snapshots/fuse_snapshot.rs index 4fd8deef6740..b2c74d042085 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_snapshots/fuse_snapshot.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_snapshots/fuse_snapshot.rs @@ -120,7 +120,7 @@ impl<'a> FuseSnapshot<'a> { DataField::new("bytes_uncompressed", u64::to_data_type()), DataField::new("bytes_compressed", u64::to_data_type()), DataField::new("index_size", u64::to_data_type()), - DataField::new_nullable("timestamp", TimestampType::new_impl(6)), + DataField::new_nullable("timestamp", TimestampType::new_impl()), ]) } } diff --git a/src/query/storages/hive/src/converters.rs b/src/query/storages/hive/src/converters.rs index 88072f1df02b..479497b4c9c8 100644 --- a/src/query/storages/hive/src/converters.rs +++ b/src/query/storages/hive/src/converters.rs @@ -174,7 +174,7 @@ fn try_from_filed_type_name(type_name: impl AsRef) -> Result "DOUBLE PRECISION" => Ok(DataTypeImpl::Float64(Float64Type::default())), // timestamp - "TIMESTAMP" => Ok(DataTypeImpl::Timestamp(TimestampType::create(3))), + "TIMESTAMP" => Ok(DataTypeImpl::Timestamp(TimestampType::create())), "DATE" => Ok(DataTypeImpl::Date(DateType::default())), _ => Err(ErrorCode::IllegalDataType(format!( diff --git a/src/query/storages/preludes/src/system/clustering_history_table.rs b/src/query/storages/preludes/src/system/clustering_history_table.rs index 688d08ad39b5..cc034a873332 100644 --- a/src/query/storages/preludes/src/system/clustering_history_table.rs +++ b/src/query/storages/preludes/src/system/clustering_history_table.rs @@ -34,8 +34,8 @@ impl SystemLogElement for ClusteringHistoryLogElement { fn schema() -> DataSchemaRef { DataSchemaRefExt::create(vec![ - DataField::new("start_time", TimestampType::new_impl(3)), - DataField::new("end_time", TimestampType::new_impl(3)), + DataField::new("start_time", TimestampType::new_impl()), + DataField::new("end_time", TimestampType::new_impl()), DataField::new("database", Vu8::to_data_type()), DataField::new("table", Vu8::to_data_type()), DataField::new("reclustered_bytes", u64::to_data_type()), diff --git a/src/query/storages/preludes/src/system/query_log_table.rs b/src/query/storages/preludes/src/system/query_log_table.rs index 5c8a82bd1164..0202fb7e25bd 100644 --- a/src/query/storages/preludes/src/system/query_log_table.rs +++ b/src/query/storages/preludes/src/system/query_log_table.rs @@ -139,8 +139,8 @@ impl SystemLogElement for QueryLogElement { DataField::new("query_kind", Vu8::to_data_type()), DataField::new("query_text", Vu8::to_data_type()), DataField::new("event_date", DateType::new_impl()), - DataField::new("event_time", TimestampType::new_impl(3)), - DataField::new("query_start_time", TimestampType::new_impl(3)), + DataField::new("event_time", TimestampType::new_impl()), + DataField::new("query_start_time", TimestampType::new_impl()), DataField::new("query_duration_ms", i64::to_data_type()), // Schema. DataField::new("current_database", Vu8::to_data_type()), diff --git a/tests/logictest/suites/base/02_function/02_0012_function_datetimes b/tests/logictest/suites/base/02_function/02_0012_function_datetimes index 40d357f37a6f..bf29fd99905b 100644 --- a/tests/logictest/suites/base/02_function/02_0012_function_datetimes +++ b/tests/logictest/suites/base/02_function/02_0012_function_datetimes @@ -75,7 +75,7 @@ select to_date('9999-12-31'); statement error 1010 -select to_date('10000-12-31'); +select to_date('10000-12-31'); statement query T select to_date('0999-12-31'); @@ -144,7 +144,7 @@ statement error 1010 select to_datetime('9999-01-01 01:12.123+02:00'); statement error 1010 -select to_datetime('10000-01-01 00:00:00'); +select to_datetime('10000-01-01 00:00:00'); statement query T select to_datetime('0999-12-31 23:59:59'); @@ -171,13 +171,13 @@ select typeof(today() - 3) = 'DATE'; 1 statement query B -select typeof(now() - 3) = 'TIMESTAMP(6)'; +select typeof(now() - 3) = 'TIMESTAMP'; ---- 1 statement query B -select typeof(to_datetime(1640019661000000)) = 'TIMESTAMP(6)'; +select typeof(to_datetime(1640019661000000)) = 'TIMESTAMP'; ---- 1 @@ -539,10 +539,10 @@ select to_start_of_week(to_date(18769), 9); 2021-05-17 statement error 1079 -select to_start_of_week(to_date('1000-01-01')); +select to_start_of_week(to_date('1000-01-01')); statement error 1079 -select to_start_of_week(to_datetime('1000-01-01 00:00:00')); +select to_start_of_week(to_datetime('1000-01-01 00:00:00')); statement query T select add_years(to_date(18321), cast(1, UINT8)); -- 2020-2-29 + 1 year @@ -607,10 +607,10 @@ select add_years(to_datetime(1582970400000000), cast(-50, INT8)); -- 2020-2-29T1 statement error 1079 -select add_years(to_date('9999-12-31'), 1); +select add_years(to_date('9999-12-31'), 1); statement error 1080 -select add_years(to_datetime('9999-12-31 23:59:59'), 1); +select add_years(to_datetime('9999-12-31 23:59:59'), 1); statement query T select subtract_months(to_date(18321), cast(13, INT16)); -- 2020-2-29 - 13 months @@ -626,10 +626,10 @@ select subtract_months(to_datetime(1582970400000000), cast(122, INT16)); -- 2020 statement error 1079 -select subtract_months(to_date('1000-01-01'), 1); +select subtract_months(to_date('1000-01-01'), 1); statement error 1080 -select subtract_months(to_datetime('1000-01-01 00:00:00'), 1); +select subtract_months(to_datetime('1000-01-01 00:00:00'), 1); statement query T select add_days(to_date(18321), cast(1, INT16)); -- 2020-2-29 + 1 day @@ -644,10 +644,10 @@ select add_days(to_datetime(1582970400000000), cast(-1, INT16)); -- 2020-2-29T10 2020-02-28 10:00:00.000000 statement error 1079 -select add_days(to_date('9999-12-31'), 1); +select add_days(to_date('9999-12-31'), 1); statement error 1080 -select add_days(to_datetime('9999-12-31 23:59:59'), 1); +select add_days(to_datetime('9999-12-31 23:59:59'), 1); statement query T select add_hours(to_datetime(1582970400000000), cast(25, INT32)); -- 2020-2-29T10:00:00 + 25 hours @@ -664,10 +664,10 @@ select add_hours(to_date(18321), cast(1.2, Float32)); statement error 1080 -select add_hours(to_date('9999-12-31'), 24); +select add_hours(to_date('9999-12-31'), 24); statement error 1080 -select add_hours(to_datetime('9999-12-31 23:59:59'), 1); +select add_hours(to_datetime('9999-12-31 23:59:59'), 1); statement query T select subtract_minutes(to_datetime(1582970400000000), cast(1, INT32)); -- 2020-2-29T10:00:00 - 1 minutes @@ -676,10 +676,10 @@ select subtract_minutes(to_datetime(1582970400000000), cast(1, INT32)); -- 2020- 2020-02-29 09:59:00.000000 statement error 1080 -select subtract_minutes(to_date('1000-01-01'), 1); +select subtract_minutes(to_date('1000-01-01'), 1); statement error 1080 -select subtract_minutes(to_datetime('1000-01-01 00:00:00'), 1); +select subtract_minutes(to_datetime('1000-01-01 00:00:00'), 1); statement query T select add_seconds(to_datetime(1582970400000000), cast(61, INT32)); -- 2020-2-29T10:00:00 + 61 seconds @@ -970,7 +970,7 @@ statement ok insert into t values('0999-04-02 15:10:28.221', '2022-04-02 15:10:28.222', '2020-10-10'); statement error 1010 -insert into t values('10000-01-01 00:00:00', '2022-04-02 15:10:28.221', '2020-10-10'); +insert into t values('10000-01-01 00:00:00', '2022-04-02 15:10:28.221', '2020-10-10'); statement ok insert into t values('2022-04-02 15:10:28.221', '2022-04-02 15:10:28.223', '0999-10-10'); @@ -983,7 +983,7 @@ insert into t values('2022-04-02T15:10:28-08:13', '2022-04-02T15:10:28.223+08:00 skipif clickhouse statement error 1010 -insert into t values('2022-04-02 15:10:28.221', '2022-04-02 15:10:28.221', '10000-10-10'); +insert into t values('2022-04-02 15:10:28.221', '2022-04-02 15:10:28.221', '10000-10-10'); statement query TTT select * from t order by b; diff --git a/tests/logictest/suites/base/05_ddl/05_0000_ddl_create_tables b/tests/logictest/suites/base/05_ddl/05_0000_ddl_create_tables index c479f72c2f6d..35586390acc2 100644 --- a/tests/logictest/suites/base/05_ddl/05_0000_ddl_create_tables +++ b/tests/logictest/suites/base/05_ddl/05_0000_ddl_create_tables @@ -175,8 +175,8 @@ bigint_unsigned BIGINT UNSIGNED NO 0 float FLOAT NO 0 double DOUBLE NO 0 date DATE NO 0 -datetime TIMESTAMP(6) NO 0 -ts TIMESTAMP(6) NO 0 +datetime TIMESTAMP NO 0 +ts TIMESTAMP NO 0 str VARCHAR NO 3 bool BOOLEAN NO false arr ARRAY NO [] @@ -201,8 +201,8 @@ bigint_unsigned BIGINT UNSIGNED NO 0 float FLOAT NO 0 double DOUBLE NO 0 date DATE NO 0 -datetime TIMESTAMP(6) NO 0 -ts TIMESTAMP(6) NO 0 +datetime TIMESTAMP NO 0 +ts TIMESTAMP NO 0 str VARCHAR NO 3 bool BOOLEAN NO false arr ARRAY NO [] diff --git a/website/blog/2022-09-14-json-datatype.md b/website/blog/2022-09-14-json-datatype.md index a38db623c719..341e23075450 100644 --- a/website/blog/2022-09-14-json-datatype.md +++ b/website/blog/2022-09-14-json-datatype.md @@ -23,12 +23,12 @@ Databend supports structured data types, as well as JSON. This post dives deeply Databend stores semi-structured data as the VARIANT (also called JSON) data type: ```sql -CREATE TABLE test - ( - id INT32, - v1 VARIANT, - v2 JSON - ); +CREATE TABLE test + ( + id INT32, + v1 VARIANT, + v2 JSON + ); ``` The JSON data needs to be generated by calling the `parse_json` or `try_parse_json` function. The input string must be in the standard JSON format, including Null, Boolean, Number, String, Array, and Object. In case of parsing failure due to invalid string, the `parse_json` function will return an error while the `try_parse_json` function will return a NULL value. @@ -139,16 +139,16 @@ The GitHub events dataset (downloaded from [GH Archive](https://www.gharchive.or From the data above, we can see that the `actor`, `repo`, `payload`, and `org` fields have a nested structure and can be stored as JSON. Others can be stored as basic data types. So we can create a table like this: ```sql -CREATE TABLE `github_data` - ( - `id` VARCHAR, - `type` VARCHAR, - `actor` JSON, - `repo` JSON, - `payload` JSON, - `public` BOOLEAN, - `created_at` timestamp(0), - `org` json +CREATE TABLE `github_data` + ( + `id` VARCHAR, + `type` VARCHAR, + `actor` JSON, + `repo` JSON, + `payload` JSON, + `public` BOOLEAN, + `created_at` timestamp, + `org` json ); ``` @@ -166,12 +166,12 @@ FILE_FORMAT = ( The following code returns the top 10 projects with the most commits: ```sql -SELECT repo:name, - count(id) -FROM github_data -WHERE type = 'PushEvent' -GROUP BY repo:name -ORDER BY count(id) DESC +SELECT repo:name, + count(id) +FROM github_data +WHERE type = 'PushEvent' +GROUP BY repo:name +ORDER BY count(id) DESC LIMIT 10; +----------------------------------------------------------+-----------+ @@ -193,12 +193,12 @@ LIMIT 10; The following code returns the top 10 users with the most forks: ```sql -SELECT actor:login, - count(id) -FROM github_data -WHERE type='ForkEvent' -GROUP BY actor:login -ORDER BY count(id) DESC +SELECT actor:login, + count(id) +FROM github_data +WHERE type='ForkEvent' +GROUP BY actor:login +ORDER BY count(id) DESC LIMIT 10; +-----------------------------------+-----------+ From 758b50c63a3ec4f71e6ca5076526ae57e3ef809e Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Wed, 12 Oct 2022 11:28:58 +0800 Subject: [PATCH 2/7] refactor(query): remove precision in timestamp type --- src/meta/proto-conv/src/data_from_to_protobuf_impl.rs | 3 +-- src/meta/proto-conv/tests/it/proto_conv.rs | 6 +++--- src/query/expression/tests/it/convert.rs | 2 +- src/query/functions-v2/tests/it/scalars/parser.rs | 2 +- src/query/storages/hive/src/converters.rs | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/meta/proto-conv/src/data_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/data_from_to_protobuf_impl.rs index 3bb46af5df99..62bd7f5f305a 100644 --- a/src/meta/proto-conv/src/data_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/data_from_to_protobuf_impl.rs @@ -359,8 +359,7 @@ impl FromToProto for dv::TimestampType { fn from_pb(p: pb::Timestamp) -> Result where Self: Sized { check_ver(p.ver, p.min_compatible)?; - let v = dv::TimestampType::create(); - Ok(v) + Ok(dv::TimestampType) } fn to_pb(&self) -> Result { diff --git a/src/meta/proto-conv/tests/it/proto_conv.rs b/src/meta/proto-conv/tests/it/proto_conv.rs index c1b26e641461..456dbc7605f7 100644 --- a/src/meta/proto-conv/tests/it/proto_conv.rs +++ b/src/meta/proto-conv/tests/it/proto_conv.rs @@ -166,7 +166,7 @@ fn new_table_meta() -> mt::TableMeta { dv::DataField::new("float32", dv::Float32Type::default().into()), dv::DataField::new("float64", dv::Float64Type::default().into()), dv::DataField::new("date", dv::DateType::default().into()), - dv::DataField::new("timestamp", dv::TimestampType::create().into()), + dv::DataField::new("timestamp", dv::TimestampType::new_impl()), dv::DataField::new("string", dv::StringType::default().into()), dv::DataField::new( "struct", @@ -232,7 +232,7 @@ fn new_table_meta_v10() -> mt::TableMeta { dv::DataField::new("float32", dv::Float32Type::default().into()), dv::DataField::new("float64", dv::Float64Type::default().into()), dv::DataField::new("date", dv::DateType::default().into()), - dv::DataField::new("timestamp", dv::TimestampType::create().into()), + dv::DataField::new("timestamp", dv::TimestampType::new_impl()), dv::DataField::new("string", dv::StringType::default().into()), dv::DataField::new( "struct", @@ -298,7 +298,7 @@ fn new_table_meta_v12() -> mt::TableMeta { dv::DataField::new("float32", dv::Float32Type::default().into()), dv::DataField::new("float64", dv::Float64Type::default().into()), dv::DataField::new("date", dv::DateType::default().into()), - dv::DataField::new("timestamp", dv::TimestampType::create().into()), + dv::DataField::new("timestamp", dv::TimestampType::new_impl()), dv::DataField::new("string", dv::StringType::default().into()), dv::DataField::new( "struct", diff --git a/src/query/expression/tests/it/convert.rs b/src/query/expression/tests/it/convert.rs index 700b0a7c99b9..72cf3beed928 100644 --- a/src/query/expression/tests/it/convert.rs +++ b/src/query/expression/tests/it/convert.rs @@ -41,7 +41,7 @@ fn random_type() -> DataTypeImpl { 12 => f64::to_data_type(), 13 => DataTypeImpl::Date(DateType {}), - 14 => DataTypeImpl::Timestamp(TimestampType::create()), + 14 => DataTypeImpl::Timestamp(TimestampType), 15 => DataTypeImpl::String(StringType {}), 16 => DataTypeImpl::Struct(StructType::create(None, vec![ f32::to_data_type(), diff --git a/src/query/functions-v2/tests/it/scalars/parser.rs b/src/query/functions-v2/tests/it/scalars/parser.rs index 0db51717356b..c66e87bb2e35 100644 --- a/src/query/functions-v2/tests/it/scalars/parser.rs +++ b/src/query/functions-v2/tests/it/scalars/parser.rs @@ -303,7 +303,7 @@ fn transform_data_type(target_type: common_ast::ast::TypeName) -> DataType { common_ast::ast::TypeName::Float32 => DataType::Number(NumberDataType::Float32), common_ast::ast::TypeName::Float64 => DataType::Number(NumberDataType::Float64), common_ast::ast::TypeName::String => DataType::String, - common_ast::ast::TypeName::Timestamp { precision: None } => DataType::Timestamp, + common_ast::ast::TypeName::Timestamp => DataType::Timestamp, common_ast::ast::TypeName::Date => DataType::Date, common_ast::ast::TypeName::Array { item_type: Some(item_type), diff --git a/src/query/storages/hive/src/converters.rs b/src/query/storages/hive/src/converters.rs index 479497b4c9c8..560fa7cb0084 100644 --- a/src/query/storages/hive/src/converters.rs +++ b/src/query/storages/hive/src/converters.rs @@ -174,7 +174,7 @@ fn try_from_filed_type_name(type_name: impl AsRef) -> Result "DOUBLE PRECISION" => Ok(DataTypeImpl::Float64(Float64Type::default())), // timestamp - "TIMESTAMP" => Ok(DataTypeImpl::Timestamp(TimestampType::create())), + "TIMESTAMP" => Ok(DataTypeImpl::Timestamp(TimestampType)), "DATE" => Ok(DataTypeImpl::Date(DateType::default())), _ => Err(ErrorCode::IllegalDataType(format!( From cb85ebd3a3e744a9c15273906a3cac87060dbcf9 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Wed, 12 Oct 2022 13:20:45 +0800 Subject: [PATCH 3/7] refactor(query): fix tests --- .../src/data_from_to_protobuf_impl.rs | 2 +- src/meta/protos/proto/datatype.proto | 1 + src/query/datavalues/src/types/data_type.rs | 2 +- .../datavalues/src/types/type_timestamp.rs | 10 ++++-- .../tests/it/types/serializations/mod.rs | 10 +++--- src/query/expression/tests/it/convert.rs | 2 +- src/query/functions/src/scalars/dates/date.rs | 3 ++ .../src/scalars/dates/interval_function.rs | 6 ++-- .../it/scalars/dates/interval_function.rs | 6 ++-- src/query/storages/hive/src/converters.rs | 2 +- .../base/02_function/02_0002_function_cast | 4 +-- .../02_function/02_0012_function_datetimes | 24 +++++++------- .../02_function/02_0012_function_datetimes_tz | 32 +++++++++---------- 13 files changed, 55 insertions(+), 49 deletions(-) diff --git a/src/meta/proto-conv/src/data_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/data_from_to_protobuf_impl.rs index 62bd7f5f305a..bd2dbafd2187 100644 --- a/src/meta/proto-conv/src/data_from_to_protobuf_impl.rs +++ b/src/meta/proto-conv/src/data_from_to_protobuf_impl.rs @@ -359,7 +359,7 @@ impl FromToProto for dv::TimestampType { fn from_pb(p: pb::Timestamp) -> Result where Self: Sized { check_ver(p.ver, p.min_compatible)?; - Ok(dv::TimestampType) + Ok(dv::TimestampType::default()) } fn to_pb(&self) -> Result { diff --git a/src/meta/protos/proto/datatype.proto b/src/meta/protos/proto/datatype.proto index 9e9a59e71752..0be591d9b238 100644 --- a/src/meta/protos/proto/datatype.proto +++ b/src/meta/protos/proto/datatype.proto @@ -62,6 +62,7 @@ message Timestamp { uint64 min_compatible = 101; /// It's deprecated + /// uint64 precision = 1; reserved 1; } diff --git a/src/query/datavalues/src/types/data_type.rs b/src/query/datavalues/src/types/data_type.rs index c84f77829726..5158734a6c25 100644 --- a/src/query/datavalues/src/types/data_type.rs +++ b/src/query/datavalues/src/types/data_type.rs @@ -188,7 +188,7 @@ pub fn from_arrow_type(dt: &ArrowType) -> DataTypeImpl { DataTypeImpl::String(StringType::default()) } - ArrowType::Timestamp(_, _) => DataTypeImpl::Timestamp(TimestampType), + ArrowType::Timestamp(_, _) => TimestampType::new_impl(), ArrowType::Date32 | ArrowType::Date64 => DataTypeImpl::Date(DateType::default()), diff --git a/src/query/datavalues/src/types/type_timestamp.rs b/src/query/datavalues/src/types/type_timestamp.rs index c13a642f4cc9..758d3f6bae03 100644 --- a/src/query/datavalues/src/types/type_timestamp.rs +++ b/src/query/datavalues/src/types/type_timestamp.rs @@ -48,11 +48,15 @@ pub fn check_timestamp(micros: i64) -> Result<()> { /// Timestamp type only stores UTC time in microseconds #[derive(Default, Clone, Hash, serde::Deserialize, serde::Serialize)] -pub struct TimestampType; +pub struct TimestampType { + // Deprecated, used as a placeholder for backward compatibility + #[serde(skip)] + precision: usize, +} impl TimestampType { pub fn new_impl() -> DataTypeImpl { - DataTypeImpl::Timestamp(TimestampType) + DataTypeImpl::Timestamp(TimestampType { precision: 0 }) } #[inline] @@ -81,7 +85,7 @@ impl DataType for TimestampType { } fn name(&self) -> String { - format!("Timestamp") + "Timestamp".to_string() } fn aliases(&self) -> &[&str] { diff --git a/src/query/datavalues/tests/it/types/serializations/mod.rs b/src/query/datavalues/tests/it/types/serializations/mod.rs index a2815a0d9786..8ef1e633fc41 100644 --- a/src/query/datavalues/tests/it/types/serializations/mod.rs +++ b/src/query/datavalues/tests/it/types/serializations/mod.rs @@ -49,14 +49,14 @@ fn test_serializers() -> Result<()> { col_str: vec!["1".to_owned(), "2".to_owned(), "1".to_owned()], }, Test { - name: "datetime32", + name: "datetime", data_type: TimestampType::new_impl(), column: Series::from_data(vec![1630320462000000i64, 1637117572000000i64, 1000000]), - val_str: "2021-08-30 10:47:42", + val_str: "2021-08-30 10:47:42.000000", col_str: vec![ - "2021-08-30 10:47:42".to_owned(), - "2021-11-17 02:52:52".to_owned(), - "1970-01-01 00:00:01".to_owned(), + "2021-08-30 10:47:42.000000".to_owned(), + "2021-11-17 02:52:52.000000".to_owned(), + "1970-01-01 00:00:01.000000".to_owned(), ], }, Test { diff --git a/src/query/expression/tests/it/convert.rs b/src/query/expression/tests/it/convert.rs index 72cf3beed928..0dc6fd588644 100644 --- a/src/query/expression/tests/it/convert.rs +++ b/src/query/expression/tests/it/convert.rs @@ -41,7 +41,7 @@ fn random_type() -> DataTypeImpl { 12 => f64::to_data_type(), 13 => DataTypeImpl::Date(DateType {}), - 14 => DataTypeImpl::Timestamp(TimestampType), + 14 => TimestampType::new_impl(), 15 => DataTypeImpl::String(StringType {}), 16 => DataTypeImpl::Struct(StructType::create(None, vec![ f32::to_data_type(), diff --git a/src/query/functions/src/scalars/dates/date.rs b/src/query/functions/src/scalars/dates/date.rs index d37e8ac4f213..595dc7c2d1bc 100644 --- a/src/query/functions/src/scalars/dates/date.rs +++ b/src/query/functions/src/scalars/dates/date.rs @@ -124,12 +124,15 @@ impl DateFunction { factory.register("add_quarters", AddQuartersFunction::desc(1)); factory.register("add_months", AddMonthsFunction::desc(1)); factory.register("add_days", AddDaysFunction::desc(1)); + factory.register("add_hours", AddTimesFunction::desc(3600)); factory.register("add_minutes", AddTimesFunction::desc(60)); factory.register("add_seconds", AddTimesFunction::desc(1)); + factory.register("subtract_years", AddYearsFunction::desc(-1)); factory.register("subtract_months", AddMonthsFunction::desc(-1)); factory.register("subtract_days", AddDaysFunction::desc(-1)); + factory.register("subtract_hours", AddTimesFunction::desc(-3600)); factory.register("subtract_minutes", AddTimesFunction::desc(-60)); factory.register("subtract_seconds", AddTimesFunction::desc(-1)); diff --git a/src/query/functions/src/scalars/dates/interval_function.rs b/src/query/functions/src/scalars/dates/interval_function.rs index e30c6989bf54..3c72b3e101c9 100644 --- a/src/query/functions/src/scalars/dates/interval_function.rs +++ b/src/query/functions/src/scalars/dates/interval_function.rs @@ -205,8 +205,7 @@ impl IntervalArithmeticImpl for AddDaysImpl { } fn eval_timestamp(l: i64, r: impl AsPrimitive, ctx: &mut EvalContext) -> i64 { - let base = 10_i64.pow(6); - let factor = ctx.factor * 24 * 3600 * base; + let factor = ctx.factor * 24 * 3600 * (1e6 as i64); (l as i64).wrapping_add(r.as_() * factor) } } @@ -223,8 +222,7 @@ impl IntervalArithmeticImpl for AddTimesImpl { } fn eval_timestamp(l: i64, r: impl AsPrimitive, ctx: &mut EvalContext) -> i64 { - let base = 10_i64.pow(6); - let factor = ctx.factor * base; + let factor = ctx.factor * 1_000_000; (l as i64).wrapping_add(r.as_() * factor) } } diff --git a/src/query/functions/tests/it/scalars/dates/interval_function.rs b/src/query/functions/tests/it/scalars/dates/interval_function.rs index ae1b744ab26a..e0f998b8f013 100644 --- a/src/query/functions/tests/it/scalars/dates/interval_function.rs +++ b/src/query/functions/tests/it/scalars/dates/interval_function.rs @@ -164,7 +164,7 @@ fn test_add_subtract_seconds() -> Result<()> { ]); let blocks = DataBlock::create(schema.clone(), vec![ - Series::from_data(vec![dt_to_seconds("2020-02-29T23:59:59Z")]), + Series::from_data(vec![dt_to_seconds("2020-02-29T23:59:59Z") * 1e6 as i64]), Series::from_data(vec![1_u8]), Series::from_data(vec![1_u16]), Series::from_data(vec![1_u32]), @@ -215,7 +215,7 @@ fn test_add_subtract_seconds() -> Result<()> { )?; assert_eq!(col.len(), 1); assert_eq!(col.data_type().data_type_id(), TypeID::Int64); - expects.push(col.get_i64(0)?); + expects.push(col.get_i64(0)? / 1e6 as i64); } assert_eq!(expects, vec![ dt_to_seconds("2020-03-01T00:00:00Z"), @@ -246,7 +246,7 @@ fn test_add_subtract_seconds() -> Result<()> { )?; assert_eq!(col.len(), 1); assert_eq!(col.data_type().data_type_id(), TypeID::Int64); - expects.push(col.get_i64(0)?); + expects.push(col.get_i64(0)? / 1e6 as i64); } assert_eq!(expects, vec![ dt_to_seconds("2020-02-29T23:59:58Z"), diff --git a/src/query/storages/hive/src/converters.rs b/src/query/storages/hive/src/converters.rs index 560fa7cb0084..070865e9aefc 100644 --- a/src/query/storages/hive/src/converters.rs +++ b/src/query/storages/hive/src/converters.rs @@ -174,7 +174,7 @@ fn try_from_filed_type_name(type_name: impl AsRef) -> Result "DOUBLE PRECISION" => Ok(DataTypeImpl::Float64(Float64Type::default())), // timestamp - "TIMESTAMP" => Ok(DataTypeImpl::Timestamp(TimestampType)), + "TIMESTAMP" => Ok(TimestampType::new_impl()), "DATE" => Ok(DataTypeImpl::Date(DateType::default())), _ => Err(ErrorCode::IllegalDataType(format!( diff --git a/tests/logictest/suites/base/02_function/02_0002_function_cast b/tests/logictest/suites/base/02_function/02_0002_function_cast index b2cb644b3515..b0b4d270654b 100644 --- a/tests/logictest/suites/base/02_function/02_0002_function_cast +++ b/tests/logictest/suites/base/02_function/02_0002_function_cast @@ -467,10 +467,10 @@ SELECT parse_json('"2022-01-01"')::date; 2022-01-01 statement query T -SELECT parse_json('"2022-01-01 01:01:01"')::datetime(0); +SELECT parse_json('"2022-01-01 01:01:01"')::datetime; ---- -2022-01-01 01:01:01 +2022-01-01 01:01:01.000000 statement error 1010 diff --git a/tests/logictest/suites/base/02_function/02_0012_function_datetimes b/tests/logictest/suites/base/02_function/02_0012_function_datetimes index bf29fd99905b..2090b5fde504 100644 --- a/tests/logictest/suites/base/02_function/02_0012_function_datetimes +++ b/tests/logictest/suites/base/02_function/02_0012_function_datetimes @@ -260,37 +260,37 @@ statement query T select time_slot(to_datetime(1630320462000000)); ---- -2021-08-30 10:30:00 +2021-08-30 10:30:00.000000 statement query T select to_start_of_hour(to_datetime(1630320462000000)); ---- -2021-08-30 10:00:00 +2021-08-30 10:00:00.000000 statement query T select to_start_of_fifteen_minutes(to_datetime(1630320462000000)); ---- -2021-08-30 10:45:00 +2021-08-30 10:45:00.000000 statement query T select to_start_of_minute(to_datetime(1630320462000000)); ---- -2021-08-30 10:47:00 +2021-08-30 10:47:00.000000 statement query T select to_start_of_five_minutes(to_datetime(1630320462000000)); ---- -2021-08-30 10:45:00 +2021-08-30 10:45:00.000000 statement query T select to_start_of_ten_minutes(to_datetime(1630320462000000)); ---- -2021-08-30 10:40:00 +2021-08-30 10:40:00.000000 statement query B select time_slot(now()) <= now(); @@ -989,12 +989,12 @@ statement query TTT select * from t order by b; ---- -2022-04-02 23:23:28.000000 2022-04-02 07:10:28.223 2022-04-02 -2022-04-02 15:10:28.000000 2022-04-02 15:10:28.000 1000-01-01 -2022-04-02 15:10:28.221000 2022-04-02 15:10:28.221 9999-12-31 -1000-01-01 00:00:00.000000 2022-04-02 15:10:28.222 2020-10-10 -2022-04-02 15:10:28.221000 2022-04-02 15:10:28.223 1000-01-01 -2022-04-02 06:57:28.000000 2022-04-02 23:10:28.223 2022-04-02 +2022-04-02 23:23:28.000000 2022-04-02 07:10:28.223000 2022-04-02 +2022-04-02 15:10:28.000000 2022-04-02 15:10:28.000000 1000-01-01 +2022-04-02 15:10:28.221000 2022-04-02 15:10:28.221000 9999-12-31 +1000-01-01 00:00:00.000000 2022-04-02 15:10:28.222000 2020-10-10 +2022-04-02 15:10:28.221000 2022-04-02 15:10:28.223000 1000-01-01 +2022-04-02 06:57:28.000000 2022-04-02 23:10:28.223000 2022-04-02 statement ok drop table t; diff --git a/tests/logictest/suites/base/02_function/02_0012_function_datetimes_tz b/tests/logictest/suites/base/02_function/02_0012_function_datetimes_tz index ef9c1f633a60..128ca9133b90 100644 --- a/tests/logictest/suites/base/02_function/02_0012_function_datetimes_tz +++ b/tests/logictest/suites/base/02_function/02_0012_function_datetimes_tz @@ -185,49 +185,49 @@ statement query T select to_start_of_second(to_timestamp(1619822911999000)); ---- -2021-04-30 22:48:31 +2021-04-30 22:48:31.000000 statement query T select to_start_of_minute(to_timestamp(1619822911999000)); ---- -2021-04-30 22:48:00 +2021-04-30 22:48:00.000000 statement query T select to_start_of_five_minutes(to_timestamp(1619822911999000)); ---- -2021-04-30 22:45:00 +2021-04-30 22:45:00.000000 statement query T select to_start_of_ten_minutes(to_timestamp(1619822911999000)); ---- -2021-04-30 22:40:00 +2021-04-30 22:40:00.000000 statement query T select to_start_of_fifteen_minutes(to_timestamp(1619822911999000)); ---- -2021-04-30 22:45:00 +2021-04-30 22:45:00.000000 statement query T select time_slot(to_timestamp(1619822911999000)); ---- -2021-04-30 22:30:00 +2021-04-30 22:30:00.000000 statement query T select to_start_of_hour(to_timestamp(1619822911999000)); ---- -2021-04-30 22:00:00 +2021-04-30 22:00:00.000000 statement query T select to_start_of_day(to_timestamp(1619822911999000)); ---- -2021-04-30 00:00:00 +2021-04-30 00:00:00.000000 statement query T select to_start_of_week(to_timestamp(1619822911999000)); @@ -242,49 +242,49 @@ statement query T select to_start_of_second(to_timestamp(1619822911999000)); ---- -2021-05-01 06:48:31 +2021-05-01 06:48:31.000000 statement query T select to_start_of_minute(to_timestamp(1619822911999000)); ---- -2021-05-01 06:48:00 +2021-05-01 06:48:00.000000 statement query T select to_start_of_five_minutes(to_timestamp(1619822911999000)); ---- -2021-05-01 06:45:00 +2021-05-01 06:45:00.000000 statement query T select to_start_of_ten_minutes(to_timestamp(1619822911999000)); ---- -2021-05-01 06:40:00 +2021-05-01 06:40:00.000000 statement query T select to_start_of_fifteen_minutes(to_timestamp(1619822911999000)); ---- -2021-05-01 06:45:00 +2021-05-01 06:45:00.000000 statement query T select time_slot(to_timestamp(1619822911999000)); ---- -2021-05-01 06:30:00 +2021-05-01 06:30:00.000000 statement query T select to_start_of_hour(to_timestamp(1619822911999000)); ---- -2021-05-01 06:00:00 +2021-05-01 06:00:00.000000 statement query T select to_start_of_day(to_timestamp(1619822911999000)); ---- -2021-05-01 00:00:00 +2021-05-01 00:00:00.000000 statement query T select to_start_of_week(to_timestamp(1619822911999000)); From ca950414de2ab0294a7da085b947b46cfbae4f47 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Wed, 12 Oct 2022 14:23:36 +0800 Subject: [PATCH 4/7] refactor(query): fix tests --- .../service/tests/it/storages/testdata/system-tables.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/service/tests/it/storages/testdata/system-tables.txt b/src/query/service/tests/it/storages/testdata/system-tables.txt index 287fb32ba5dc..faae2afa09d9 100644 --- a/src/query/service/tests/it/storages/testdata/system-tables.txt +++ b/src/query/service/tests/it/storages/testdata/system-tables.txt @@ -46,11 +46,11 @@ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemCo | dropped_on | system | tables | VARCHAR | | | false | | | dropped_on | system | tables_with_history | VARCHAR | | | false | | | dummy | system | one | TINYINT UNSIGNED | | | false | | -| end_time | system | clustering_history | TIMESTAMP(3) | | | false | | +| end_time | system | clustering_history | TIMESTAMP | | | false | | | engine | system | tables | VARCHAR | | | false | | | engine | system | tables_with_history | VARCHAR | | | false | | | event_date | system | query_log | DATE | | | false | | -| event_time | system | query_log | TIMESTAMP(3) | | | false | | +| event_time | system | query_log | TIMESTAMP | | | false | | | example | system | functions | VARCHAR | | | false | | | exception_code | system | query_log | INT | | | false | | | exception_text | system | query_log | VARCHAR | | | false | | @@ -104,7 +104,7 @@ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemCo | query_duration_ms | system | query_log | BIGINT | | | false | | | query_id | system | query_log | VARCHAR | | | false | | | query_kind | system | query_log | VARCHAR | | | false | | -| query_start_time | system | query_log | TIMESTAMP(3) | | | false | | +| query_start_time | system | query_log | TIMESTAMP | | | false | | | query_text | system | query_log | VARCHAR | | | false | | | reclustered_bytes | system | clustering_history | BIGINT UNSIGNED | | | false | | | reclustered_rows | system | clustering_history | BIGINT UNSIGNED | | | false | | @@ -125,7 +125,7 @@ DB.Table: 'system'.'columns', Table: columns-table_id:1, ver:0, Engine: SystemCo | stack_trace | system | query_log | VARCHAR | | | false | | | stage_params | system | stages | VARCHAR | | | false | | | stage_type | system | stages | VARCHAR | | | false | | -| start_time | system | clustering_history | TIMESTAMP(3) | | | false | | +| start_time | system | clustering_history | TIMESTAMP | | | false | | | state | system | processes | VARCHAR | | | false | | | syntax | system | functions | VARCHAR | | | false | | | table | system | clustering_history | VARCHAR | | | false | | From 8e02dfb5019a47d42a6079c9262cbfe70acf686c Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Wed, 12 Oct 2022 14:48:57 +0800 Subject: [PATCH 5/7] refactor(query): fix tests --- src/query/functions/tests/it/scalars/expressions.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/query/functions/tests/it/scalars/expressions.rs b/src/query/functions/tests/it/scalars/expressions.rs index fab446ffa7f0..d3d20c773001 100644 --- a/src/query/functions/tests/it/scalars/expressions.rs +++ b/src/query/functions/tests/it/scalars/expressions.rs @@ -125,7 +125,10 @@ fn test_datetime_cast_function() -> Result<()> { Series::from_data(vec![1614906061000000i64, 1635070210000000]), DataField::new("dummy_1", TimestampType::new_impl()), )], - expect: Series::from_data(vec!["2021-03-05 01:01:01", "2021-10-24 10:10:10"]), + expect: Series::from_data(vec![ + "2021-03-05 01:01:01.000000", + "2021-10-24 10:10:10.000000", + ]), error: "", }), ]; From b6add5a600e52807546b58de23a75a20ef6d24a7 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Wed, 12 Oct 2022 20:12:46 +0800 Subject: [PATCH 6/7] refactor(query): add tests --- src/meta/proto-conv/tests/it/datatype.rs | 39 ++++++++++++++++++++++++ src/meta/proto-conv/tests/it/main.rs | 1 + 2 files changed, 40 insertions(+) create mode 100644 src/meta/proto-conv/tests/it/datatype.rs diff --git a/src/meta/proto-conv/tests/it/datatype.rs b/src/meta/proto-conv/tests/it/datatype.rs new file mode 100644 index 000000000000..22182930e39a --- /dev/null +++ b/src/meta/proto-conv/tests/it/datatype.rs @@ -0,0 +1,39 @@ +// Copyright 2022 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test UserStageInfo + +use common_datavalues::DataTypeImpl; +use common_datavalues::TimestampType; + +use crate::common; + +#[test] +fn test_datatype_latest() -> anyhow::Result<()> { + common::test_pb_from_to("datatype", test_datatype())?; + Ok(()) +} + +#[test] +fn test_datatype_v15() -> anyhow::Result<()> { + // It is generated with common::test_pb_from_to. + let datatype_v15 = vec![114, 6, 160, 6, 15, 168, 6, 1, 160, 6, 15, 168, 6, 1]; + let want = TimestampType::new_impl(); + common::test_load_old(func_name!(), datatype_v15.as_slice(), want)?; + Ok(()) +} + +fn test_datatype() -> DataTypeImpl { + TimestampType::new_impl() +} diff --git a/src/meta/proto-conv/tests/it/main.rs b/src/meta/proto-conv/tests/it/main.rs index 734cc14b5cdb..b3a3054f9176 100644 --- a/src/meta/proto-conv/tests/it/main.rs +++ b/src/meta/proto-conv/tests/it/main.rs @@ -14,6 +14,7 @@ #[macro_use] pub(crate) mod common; +mod datatype; mod proto_conv; mod user_proto_conv; mod user_stage; From 6d135bd7cff3a180d60d3fce892b840eca8d6997 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Wed, 12 Oct 2022 20:43:06 +0800 Subject: [PATCH 7/7] refactor(query): fix incorrect number width --- .../fuse-meta/src/meta/v1/snapshot.rs | 24 +++++++++---------- .../fuse_snapshots/fuse_snapshot.rs | 12 +++++----- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/query/storages/fuse-meta/src/meta/v1/snapshot.rs b/src/query/storages/fuse-meta/src/meta/v1/snapshot.rs index cee0a4a07211..d0e52421517a 100644 --- a/src/query/storages/fuse-meta/src/meta/v1/snapshot.rs +++ b/src/query/storages/fuse-meta/src/meta/v1/snapshot.rs @@ -133,12 +133,12 @@ pub struct TableSnapshotLite { pub snapshot_id: SnapshotId, pub timestamp: Option>, pub prev_snapshot_id: Option<(SnapshotId, FormatVersion)>, - pub row_count: u32, - pub block_count: u32, - pub index_size: u32, - pub uncompressed_byte_size: u32, - pub compressed_byte_size: u32, - pub segment_count: u32, + pub row_count: u64, + pub block_count: u64, + pub index_size: u64, + pub uncompressed_byte_size: u64, + pub compressed_byte_size: u64, + pub segment_count: u64, } impl From<&TableSnapshot> for TableSnapshotLite { @@ -148,12 +148,12 @@ impl From<&TableSnapshot> for TableSnapshotLite { snapshot_id: value.snapshot_id, timestamp: value.timestamp, prev_snapshot_id: value.prev_snapshot_id, - row_count: value.summary.row_count as u32, - block_count: value.summary.block_count as u32, - index_size: value.summary.index_size as u32, - uncompressed_byte_size: value.summary.uncompressed_byte_size as u32, - segment_count: value.segments.len() as u32, - compressed_byte_size: value.summary.compressed_byte_size as u32, + row_count: value.summary.row_count, + block_count: value.summary.block_count, + index_size: value.summary.index_size, + uncompressed_byte_size: value.summary.uncompressed_byte_size, + segment_count: value.segments.len() as u64, + compressed_byte_size: value.summary.compressed_byte_size, } } } diff --git a/src/query/storages/fuse/src/table_functions/fuse_snapshots/fuse_snapshot.rs b/src/query/storages/fuse/src/table_functions/fuse_snapshots/fuse_snapshot.rs index b2ae58a2ea28..8b1c34297cca 100644 --- a/src/query/storages/fuse/src/table_functions/fuse_snapshots/fuse_snapshot.rs +++ b/src/query/storages/fuse/src/table_functions/fuse_snapshots/fuse_snapshot.rs @@ -62,12 +62,12 @@ impl<'a> FuseSnapshot<'a> { let mut snapshot_locations: Vec> = Vec::with_capacity(len); let mut prev_snapshot_ids: Vec>> = Vec::with_capacity(len); let mut format_versions: Vec = Vec::with_capacity(len); - let mut segment_count: Vec = Vec::with_capacity(len); - let mut block_count: Vec = Vec::with_capacity(len); - let mut row_count: Vec = Vec::with_capacity(len); - let mut compressed: Vec = Vec::with_capacity(len); - let mut uncompressed: Vec = Vec::with_capacity(len); - let mut index_size: Vec = Vec::with_capacity(len); + let mut segment_count: Vec = Vec::with_capacity(len); + let mut block_count: Vec = Vec::with_capacity(len); + let mut row_count: Vec = Vec::with_capacity(len); + let mut compressed: Vec = Vec::with_capacity(len); + let mut uncompressed: Vec = Vec::with_capacity(len); + let mut index_size: Vec = Vec::with_capacity(len); let mut timestamps: Vec> = Vec::with_capacity(len); let mut current_snapshot_version = latest_snapshot_version; for s in snapshots {