Skip to content

YQL-17502 Bigdate from/to string conversions #1447

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 31 commits into from
Feb 12, 2024
Merged

YQL-17502 Bigdate from/to string conversions #1447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
7989abf
Find ya tool both in Arcadia and github
amikish Jan 11, 2024
c67bc1b
Rebase
amikish Jan 15, 2024
823a92c
Date32 from/to integer conversions
amikish Jan 19, 2024
da35dd1
Add canonical tests
amikish Jan 21, 2024
9ac72f2
Canonize bigdate tests
amikish Jan 22, 2024
13d4b0b
Add Date32 value constructor
amikish Jan 22, 2024
f7562cd
Canonize const_date32.sql
amikish Jan 22, 2024
bb78c14
Cleanup
amikish Jan 23, 2024
d07c4dc
Conversions table
amikish Jan 23, 2024
bcf7153
Convert from/to integers
amikish Jan 24, 2024
637cf88
Some tests
amikish Jan 24, 2024
cf969a2
To string conversion
amikish Jan 24, 2024
7fd5d94
Re-use Interval from/to string conversion
amikish Jan 24, 2024
d6a3b75
Forbid IsLeapYear(0)
amikish Jan 26, 2024
6765aa7
Tests passed
amikish Jan 30, 2024
9704c9c
Fix ParseTime
amikish Jan 30, 2024
fd12247
Fix timezone offset
amikish Jan 30, 2024
55564e3
Register from integer conversions
amikish Jan 30, 2024
28c7417
Check value is in range after parsing
amikish Jan 30, 2024
679256e
Add tests for datetime64
amikish Jan 30, 2024
ddcd0cd
Check max days to prevent ui64 overflow
amikish Jan 31, 2024
fb98199
Revert ui.sh
amikish Jan 31, 2024
bea4570
Pack bigdate types
amikish Feb 4, 2024
d23963d
Canonize
amikish Feb 4, 2024
b42441e
Enable yt only for bigdate tests
amikish Feb 5, 2024
d8e86d9
Fail on unexpected trailing symbols
amikish Feb 5, 2024
4c23756
Mute all bigdate tests
amikish Feb 7, 2024
2b03c14
Remove temporary tests
amikish Feb 7, 2024
5d1938e
Remove new tests
amikish Feb 9, 2024
483f31b
Add line numbers for easy diff preview
amikish Feb 9, 2024
d770b46
Canonize test_sql_negative.test_weak_field-weak_field_list_type
amikish Feb 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions ydb/library/yql/core/sql_types/simple_types.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ std::optional<std::string_view> LookupSimpleTypeBySqlAlias(const std::string_vie
{"tztimestamp", "TzTimestamp"},

{"date32", "Date32"},
{"datetime64", "Datetime64"},
{"timestamp64", "Timestamp64"},
{"interval64", "Interval64"},
};

// new types (or aliases) should be added here
Expand Down
1 change: 1 addition & 0 deletions ydb/library/yql/core/yql_expr_type_annotation.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5119,6 +5119,7 @@ TMaybe<ui32> GetDataFixedSize(const TTypeAnnotationNode* typeAnnotation) {
}

if (EDataSlot::Timestamp == dataSlot || EDataSlot::Uint64 == dataSlot || EDataSlot::Int64 == dataSlot
|| EDataSlot::Datetime64 == dataSlot || EDataSlot::Timestamp64 == dataSlot || EDataSlot::Interval64 == dataSlot
|| EDataSlot::Double == dataSlot || EDataSlot::Interval == dataSlot) {
return 8;
}
Expand Down
3 changes: 3 additions & 0 deletions ydb/library/yql/core/yql_opt_utils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1578,6 +1578,9 @@ ui64 GetTypeWeight(const TTypeAnnotationNode& type) {
case NUdf::EDataSlot::Int64:
case NUdf::EDataSlot::Uint64:
case NUdf::EDataSlot::Double:
case NUdf::EDataSlot::Datetime64:
case NUdf::EDataSlot::Timestamp64:
case NUdf::EDataSlot::Interval64:
case NUdf::EDataSlot::Timestamp:
case NUdf::EDataSlot::Interval: return 8;

Expand Down
6 changes: 6 additions & 0 deletions ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,9 @@ NUdf::TUnboxedValue UnpackFromChunkedBuffer(const TType* type, TChunkedInputBuff
case NUdf::EDataSlot::Timestamp:
return NUdf::TUnboxedValuePod(UnpackData<Fast, ui64>(buf));
case NUdf::EDataSlot::Interval:
case NUdf::EDataSlot::Datetime64:
case NUdf::EDataSlot::Timestamp64:
case NUdf::EDataSlot::Interval64:
return NUdf::TUnboxedValuePod(UnpackData<Fast, i64>(buf));
case NUdf::EDataSlot::TzDate: {
auto ret = NUdf::TUnboxedValuePod(UnpackData<Fast, ui16>(buf));
Expand Down Expand Up @@ -640,6 +643,9 @@ void PackImpl(const TType* type, TBuf& buffer, const NUdf::TUnboxedValuePod& val
PackData<Fast>(value.Get<ui64>(), buffer);
break;
case NUdf::EDataSlot::Interval:
case NUdf::EDataSlot::Datetime64:
case NUdf::EDataSlot::Timestamp64:
case NUdf::EDataSlot::Interval64:
PackData<Fast>(value.Get<i64>(), buffer);
break;
case NUdf::EDataSlot::Uuid: {
Expand Down
64 changes: 58 additions & 6 deletions ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -882,6 +882,15 @@ void RegisterWideToBigDateCasts(IBuiltinFunctionRegistry& registry) {
RegisterWideToBigDateCastsImpl<NUdf::TDataType<i64>, NUdf::TDataType<NUdf::TDate32>, NUdf::MAX_DATE32, NUdf::MIN_DATE32>(registry);
RegisterWideToDateCastsImpl<NUdf::TDataType<ui32>, NUdf::TDataType<NUdf::TDate32>, NUdf::MAX_DATE32>(registry);
RegisterWideToDateCastsImpl<NUdf::TDataType<ui64>, NUdf::TDataType<NUdf::TDate32>, NUdf::MAX_DATE32>(registry);

RegisterWideToBigDateCastsImpl<NUdf::TDataType<i64>, NUdf::TDataType<NUdf::TDatetime64>, NUdf::MAX_DATETIME64, NUdf::MIN_DATETIME64>(registry);
RegisterWideToDateCastsImpl<NUdf::TDataType<ui64>, NUdf::TDataType<NUdf::TDatetime64>, NUdf::MAX_DATETIME64>(registry);

RegisterWideToBigDateCastsImpl<NUdf::TDataType<i64>, NUdf::TDataType<NUdf::TTimestamp64>, NUdf::MAX_TIMESTAMP64, NUdf::MIN_TIMESTAMP64>(registry);
RegisterWideToDateCastsImpl<NUdf::TDataType<ui64>, NUdf::TDataType<NUdf::TTimestamp64>, NUdf::MAX_TIMESTAMP64>(registry);

RegisterWideToBigDateCastsImpl<NUdf::TDataType<i64>, NUdf::TDataType<NUdf::TInterval64>, NUdf::MAX_INTERVAL64, -NUdf::MAX_INTERVAL64>(registry);
RegisterWideToDateCastsImpl<NUdf::TDataType<ui64>, NUdf::TDataType<NUdf::TInterval64>, NUdf::MAX_INTERVAL64>(registry);
}

void RegisterWideToTimestampCasts(IBuiltinFunctionRegistry& registry) {
Expand Down Expand Up @@ -937,11 +946,6 @@ void RegisterWideToShortIntegralCasts(IBuiltinFunctionRegistry& registry) {
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDate>, NUdf::TDataType<ui8>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDate>, NUdf::TDataType<i16>>(registry);

RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDate32>, NUdf::TDataType<i8>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDate32>, NUdf::TDataType<ui8>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDate32>, NUdf::TDataType<i16>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDate32>, NUdf::TDataType<ui16>>(registry);

RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TTzDate>, NUdf::TDataType<i8>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TTzDate>, NUdf::TDataType<ui8>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TTzDate>, NUdf::TDataType<i16>>(registry);
Expand Down Expand Up @@ -977,6 +981,29 @@ void RegisterWideToShortIntegralCasts(IBuiltinFunctionRegistry& registry) {
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TInterval>, NUdf::TDataType<i32>>(registry);
RegisterWideToUnsignedCasts<NUdf::TDataType<NUdf::TInterval>>(registry);

RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDate32>, NUdf::TDataType<i8>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDate32>, NUdf::TDataType<i16>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDate32>, NUdf::TDataType<i32>>(registry);
RegisterWideToUnsignedCasts<NUdf::TDataType<NUdf::TDate32>>(registry);

RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDatetime64>, NUdf::TDataType<i8>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDatetime64>, NUdf::TDataType<i16>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDatetime64>, NUdf::TDataType<i32>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TDatetime64>, NUdf::TDataType<i64>>(registry);
RegisterWideToUnsignedCasts<NUdf::TDataType<NUdf::TDatetime64>>(registry);

RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TTimestamp64>, NUdf::TDataType<i8>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TTimestamp64>, NUdf::TDataType<i16>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TTimestamp64>, NUdf::TDataType<i32>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TTimestamp64>, NUdf::TDataType<i64>>(registry);
RegisterWideToUnsignedCasts<NUdf::TDataType<NUdf::TTimestamp64>>(registry);

RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TInterval64>, NUdf::TDataType<i8>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TInterval64>, NUdf::TDataType<i16>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TInterval64>, NUdf::TDataType<i32>>(registry);
RegisterWideToShortCastsImpl<NUdf::TDataType<NUdf::TInterval64>, NUdf::TDataType<i64>>(registry);
RegisterWideToUnsignedCasts<NUdf::TDataType<NUdf::TInterval64>>(registry);

RegisterWideToDateCasts(registry);
RegisterWideToBigDateCasts(registry);
RegisterWideToDatetimeCasts(registry);
Expand Down Expand Up @@ -1034,6 +1061,27 @@ void RegisterToBigDateConvert(IBuiltinFunctionRegistry& registry) {
RegisterConvert<NUdf::TDataType<ui8>, NUdf::TDataType<NUdf::TDate32>>(registry);
RegisterConvert<NUdf::TDataType<i16>, NUdf::TDataType<NUdf::TDate32>>(registry);
RegisterConvert<NUdf::TDataType<ui16>, NUdf::TDataType<NUdf::TDate32>>(registry);

RegisterConvert<NUdf::TDataType<i8>, NUdf::TDataType<NUdf::TDatetime64>>(registry);
RegisterConvert<NUdf::TDataType<ui8>, NUdf::TDataType<NUdf::TDatetime64>>(registry);
RegisterConvert<NUdf::TDataType<i16>, NUdf::TDataType<NUdf::TDatetime64>>(registry);
RegisterConvert<NUdf::TDataType<ui16>, NUdf::TDataType<NUdf::TDatetime64>>(registry);
RegisterConvert<NUdf::TDataType<i32>, NUdf::TDataType<NUdf::TDatetime64>>(registry);
RegisterConvert<NUdf::TDataType<ui32>, NUdf::TDataType<NUdf::TDatetime64>>(registry);

RegisterConvert<NUdf::TDataType<i8>, NUdf::TDataType<NUdf::TTimestamp64>>(registry);
RegisterConvert<NUdf::TDataType<ui8>, NUdf::TDataType<NUdf::TTimestamp64>>(registry);
RegisterConvert<NUdf::TDataType<i16>, NUdf::TDataType<NUdf::TTimestamp64>>(registry);
RegisterConvert<NUdf::TDataType<ui16>, NUdf::TDataType<NUdf::TTimestamp64>>(registry);
RegisterConvert<NUdf::TDataType<i32>, NUdf::TDataType<NUdf::TTimestamp64>>(registry);
RegisterConvert<NUdf::TDataType<ui32>, NUdf::TDataType<NUdf::TTimestamp64>>(registry);

RegisterConvert<NUdf::TDataType<i8>, NUdf::TDataType<NUdf::TInterval64>>(registry);
RegisterConvert<NUdf::TDataType<ui8>, NUdf::TDataType<NUdf::TInterval64>>(registry);
RegisterConvert<NUdf::TDataType<i16>, NUdf::TDataType<NUdf::TInterval64>>(registry);
RegisterConvert<NUdf::TDataType<ui16>, NUdf::TDataType<NUdf::TInterval64>>(registry);
RegisterConvert<NUdf::TDataType<i32>, NUdf::TDataType<NUdf::TInterval64>>(registry);
RegisterConvert<NUdf::TDataType<ui32>, NUdf::TDataType<NUdf::TInterval64>>(registry);
}

template <typename TInput, typename TOutput, bool Tz = false>
Expand Down Expand Up @@ -1143,14 +1191,18 @@ void RegisterConvert(IBuiltinFunctionRegistry& registry) {
RegisterStringConvert<NUdf::TDataType<NUdf::TJson>, NUdf::TDataType<NUdf::TUtf8>>(registry);

RegisterFromDateConvert<NUdf::TDataType<NUdf::TDate>>(registry);
RegisterFromDateConvert<NUdf::TDataType<NUdf::TDate32>>(registry);
RegisterFromDateConvert<NUdf::TDataType<NUdf::TDatetime>>(registry);
RegisterFromDateConvert<NUdf::TDataType<NUdf::TTimestamp>>(registry);
RegisterFromDateConvert<NUdf::TDataType<NUdf::TInterval>>(registry);
RegisterFromDateConvert<NUdf::TDataType<NUdf::TTzDate>>(registry);
RegisterFromDateConvert<NUdf::TDataType<NUdf::TTzDatetime>>(registry);
RegisterFromDateConvert<NUdf::TDataType<NUdf::TTzTimestamp>>(registry);

RegisterFromDateConvert<NUdf::TDataType<NUdf::TDate32>>(registry);
RegisterFromDateConvert<NUdf::TDataType<NUdf::TDatetime64>>(registry);
RegisterFromDateConvert<NUdf::TDataType<NUdf::TTimestamp64>>(registry);
RegisterFromDateConvert<NUdf::TDataType<NUdf::TInterval64>>(registry);

RegisterTzDateimeConvert(registry);
RegisterDatetimeRescale(registry);
RegisterToDateConvert(registry);
Expand Down
Loading