Skip to content

Commit 4bcab84

Browse files
authored
Initial suppoprt Timestamp64/Interval64 (#3445)
1 parent 746b356 commit 4bcab84

File tree

26 files changed

+218
-6
lines changed

26 files changed

+218
-6
lines changed

ydb/core/engine/mkql_proto.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,10 @@ bool CellsFromTuple(const NKikimrMiniKQL::TType* tupleType,
125125
CASE_SIMPLE_TYPE(Datetime, ui32, Uint32);
126126
CASE_SIMPLE_TYPE(Timestamp, ui64, Uint64);
127127
CASE_SIMPLE_TYPE(Interval, i64, Int64);
128+
CASE_SIMPLE_TYPE(Date32, i32, Int32);
129+
CASE_SIMPLE_TYPE(Datetime64, i64, Int64);
130+
CASE_SIMPLE_TYPE(Timestamp64, i64, Int64);
131+
CASE_SIMPLE_TYPE(Interval64, i64, Int64);
128132

129133

130134
#undef CASE_SIMPLE_TYPE
@@ -228,6 +232,7 @@ bool CellToValue(NScheme::TTypeInfo type, const TCell& c, NKikimrMiniKQL::TValue
228232
val.MutableOptional()->SetUint32(ReadUnaligned<ui16>(c.Data()));
229233
break;
230234

235+
case NScheme::NTypeIds::Date32:
231236
case NScheme::NTypeIds::Int32:
232237
Y_ABORT_UNLESS(c.Size() == sizeof(i32));
233238
val.MutableOptional()->SetInt32(ReadUnaligned<i32>(c.Data()));
@@ -274,6 +279,9 @@ bool CellToValue(NScheme::TTypeInfo type, const TCell& c, NKikimrMiniKQL::TValue
274279
val.MutableOptional()->SetUint64(ReadUnaligned<ui64>(c.Data()));
275280
break;
276281
case NScheme::NTypeIds::Interval:
282+
case NScheme::NTypeIds::Interval64:
283+
case NScheme::NTypeIds::Timestamp64:
284+
case NScheme::NTypeIds::Datetime64:
277285
Y_ABORT_UNLESS(c.Size() == sizeof(i64));
278286
val.MutableOptional()->SetInt64(ReadUnaligned<i64>(c.Data()));
279287
break;

ydb/core/formats/arrow/converter.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,15 @@ static std::shared_ptr<arrow::Array> InplaceConvertColumn(const std::shared_ptr<
182182
newData->type = arrow::timestamp(arrow::TimeUnit::MICRO);
183183
return std::make_shared<arrow::TimestampArray>(newData);
184184
}
185+
case NScheme::NTypeIds::Timestamp64:
186+
case NScheme::NTypeIds::Interval64: {
187+
Y_ABORT_UNLESS(arrow::is_primitive(column->type()->id()));
188+
Y_ABORT_UNLESS(arrow::bit_width(column->type()->id()) == 64);
189+
190+
auto newData = column->data()->Copy();
191+
newData->type = arrow::int64();
192+
return std::make_shared<arrow::NumericArray<arrow::Int64Type>>(newData);
193+
}
185194
default:
186195
return {};
187196
}
@@ -231,6 +240,8 @@ bool TArrowToYdbConverter::NeedInplaceConversion(const NScheme::TTypeInfo& typeI
231240
case NScheme::NTypeIds::Datetime:
232241
return typeInRequest.GetTypeId() == NScheme::NTypeIds::Int32;
233242
case NScheme::NTypeIds::Timestamp:
243+
case NScheme::NTypeIds::Timestamp64:
244+
case NScheme::NTypeIds::Interval64:
234245
return typeInRequest.GetTypeId() == NScheme::NTypeIds::Int64;
235246
default:
236247
break;

ydb/core/formats/arrow/switch/switch_type.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,10 @@ template <typename TFunc>
165165
case NScheme::NTypeIds::Decimal:
166166
return callback(TTypeWrapper<arrow::Decimal128Type>());
167167

168+
case NScheme::NTypeIds::Timestamp64:
169+
case NScheme::NTypeIds::Interval64:
170+
return callback(TTypeWrapper<arrow::Int64Type>());
171+
168172
case NScheme::NTypeIds::PairUi64Ui64:
169173
case NScheme::NTypeIds::ActorId:
170174
case NScheme::NTypeIds::StepOrderId:
@@ -210,6 +214,8 @@ inline bool IsPrimitiveYqlType(const NScheme::TTypeInfo& typeInfo) {
210214
case NScheme::NTypeIds::Double:
211215
case NScheme::NTypeIds::Timestamp:
212216
case NScheme::NTypeIds::Interval:
217+
case NScheme::NTypeIds::Timestamp64:
218+
case NScheme::NTypeIds::Interval64:
213219
return true;
214220
default:
215221
break;

ydb/core/kqp/runtime/kqp_scan_data.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ TBytesStatistics GetUnboxedValueSize(const NUdf::TUnboxedValue& value, const NSc
4545
case NTypeIds::Datetime:
4646
case NTypeIds::Timestamp:
4747
case NTypeIds::Interval:
48+
case NTypeIds::Date32:
49+
case NTypeIds::Datetime64:
50+
case NTypeIds::Timestamp64:
51+
case NTypeIds::Interval64:
4852
case NTypeIds::ActorId:
4953
case NTypeIds::StepOrderId:
5054
{
@@ -330,11 +334,15 @@ TBytesStatistics WriteColumnValuesFromArrowImpl(TAccessor editAccessor,
330334
{
331335
return WriteColumnValuesFromArrowSpecImpl<TElementAccessor<arrow::Int16Array>>(editAccessor, batch, columnIndex, columnPtr, columnType);
332336
}
337+
case NTypeIds::Date32:
333338
case NTypeIds::Int32:
334339
{
335340
return WriteColumnValuesFromArrowSpecImpl<TElementAccessor<arrow::Int32Array>>(editAccessor, batch, columnIndex, columnPtr, columnType);
336341
}
337342
case NTypeIds::Int64:
343+
case NTypeIds::Timestamp64:
344+
case NTypeIds::Interval64:
345+
case NTypeIds::Datetime64:
338346
{
339347
return WriteColumnValuesFromArrowSpecImpl<TElementAccessor<arrow::Int64Array, i64>>(editAccessor, batch, columnIndex, columnPtr, columnType);
340348
}

ydb/core/kqp/ut/common/columnshard.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,10 @@ namespace NKqp {
216216
return arrow::field(name, arrow::timestamp(arrow::TimeUnit::TimeUnit::MICRO), nullable);
217217
case NScheme::NTypeIds::Interval:
218218
return arrow::field(name, arrow::duration(arrow::TimeUnit::TimeUnit::MICRO), nullable);
219+
case NScheme::NTypeIds::Timestamp64:
220+
return arrow::field(name, arrow::int64(), nullable);
221+
case NScheme::NTypeIds::Interval64:
222+
return arrow::field(name, arrow::int64(), nullable);
219223
case NScheme::NTypeIds::JsonDocument:
220224
return arrow::field(name, arrow::binary(), nullable);
221225
case NScheme::NTypeIds::Pg:

ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5559,6 +5559,34 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) {
55595559
}
55605560
testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;#;[\"test_res_1\"]]]");
55615561
}
5562+
5563+
Y_UNIT_TEST(UseTime64Columns) {
5564+
TKikimrSettings runnerSettings;
5565+
runnerSettings.WithSampleTables = false;
5566+
TTestHelper testHelper(runnerSettings);
5567+
5568+
TVector<TTestHelper::TColumnSchema> schema = {
5569+
TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false),
5570+
TTestHelper::TColumnSchema().SetName("interval").SetType(NScheme::NTypeIds::Interval64),
5571+
TTestHelper::TColumnSchema().SetName("timestamp").SetType(NScheme::NTypeIds::Timestamp64)
5572+
};
5573+
5574+
Tests::NCommon::TLoggerInit(testHelper.GetKikimr()).Initialize();
5575+
TTestHelper::TColumnTable testTable;
5576+
5577+
testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"id"}).SetSharding({"id"}).SetSchema(schema);
5578+
testHelper.CreateTable(testTable);
5579+
5580+
{
5581+
TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema));
5582+
tableInserter.AddRow().Add(1).Add(123).Add(-456);
5583+
tableInserter.AddRow().Add(2).Add(-789).AddNull();
5584+
testHelper.BulkUpsert(testTable, tableInserter);
5585+
}
5586+
5587+
testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;[123];[-456]]]");
5588+
testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=2", "[[2;[-789];#]]");
5589+
}
55625590
/*
55635591
Y_UNIT_TEST(AddColumnOnSchemeChange) {
55645592
TKikimrSettings runnerSettings;

ydb/core/local_pgwire/local_pgwire_util.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,14 @@ TString ColumnPrimitiveValueToString(NYdb::TValueParser& valueParser) {
3838
return valueParser.GetTimestamp().ToString();
3939
case NYdb::EPrimitiveType::Interval:
4040
return TStringBuilder() << valueParser.GetInterval();
41+
case NYdb::EPrimitiveType::Date32:
42+
return TStringBuilder() << valueParser.GetDate32();
43+
case NYdb::EPrimitiveType::Datetime64:
44+
return TStringBuilder() << valueParser.GetDatetime64();
45+
case NYdb::EPrimitiveType::Timestamp64:
46+
return TStringBuilder() << valueParser.GetTimestamp64();
47+
case NYdb::EPrimitiveType::Interval64:
48+
return TStringBuilder() << valueParser.GetInterval64();
4149
case NYdb::EPrimitiveType::TzDate:
4250
return valueParser.GetTzDate();
4351
case NYdb::EPrimitiveType::TzDatetime:

ydb/core/persqueue/type_codecs_defs.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ TTypeCodecs::TTypeCodecs(TTypeId typeId) {
7070
AddCodec<TVarLenCodec<false>>();
7171

7272
switch (typeId) {
73+
case NTypeIds::Date32:
7374
case NTypeIds::Int32:
7475
AddFixedLen<TInt32>(this);
7576
AddIntCodecs<TInt32>(this);
@@ -79,6 +80,9 @@ TTypeCodecs::TTypeCodecs(TTypeId typeId) {
7980
AddIntCodecs<TUint32>(this);
8081
break;
8182
case NTypeIds::Int64:
83+
case NTypeIds::Datetime64:
84+
case NTypeIds::Timestamp64:
85+
case NTypeIds::Interval64:
8286
AddFixedLen<TInt64>(this);
8387
AddIntCodecs<TInt64>(this);
8488
break;

ydb/core/scheme_types/scheme_type_registry.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ TTypeRegistry::TTypeRegistry()
4040
RegisterType<TInterval>();
4141
RegisterType<TDyNumber>();
4242
RegisterType<TUuid>();
43+
RegisterType<TTimestamp64>();
44+
RegisterType<TInterval64>();
4345
}
4446

4547
void TTypeRegistry::CalculateMetadataEtag() {

ydb/core/scheme_types/scheme_types_defs.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ namespace NNames {
3434
DECLARE_TYPED_TYPE_NAME(Datetime);
3535
DECLARE_TYPED_TYPE_NAME(Timestamp);
3636
DECLARE_TYPED_TYPE_NAME(Interval);
37+
DECLARE_TYPED_TYPE_NAME(Timestamp64);
38+
DECLARE_TYPED_TYPE_NAME(Interval64);
3739

3840
DECLARE_TYPED_TYPE_NAME(DyNumber);
3941
DECLARE_TYPED_TYPE_NAME(Uuid);

0 commit comments

Comments
 (0)