Skip to content

Commit 6e1e9e4

Browse files
vitstnvitstn
authored
More cases for data->pg conversions (#5118)
1 parent 04e0301 commit 6e1e9e4

File tree

13 files changed

+179
-129
lines changed

13 files changed

+179
-129
lines changed

‎ydb/library/yql/core/type_ann/type_ann_pg.cpp‎

Lines changed: 6 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -394,38 +394,14 @@ IGraphTransformer::TStatus PgCallWrapper(const TExprNode::TPtr& input, TExprNode
394394
}
395395

396396
const TTypeAnnotationNode* FromPgImpl(TPositionHandle pos, const TTypeAnnotationNode* type, TExprContext& ctx) {
397-
auto name = type->Cast<TPgExprType>()->GetName();
398-
const TDataExprType* dataType;
399-
if (name == "bool") {
400-
dataType = ctx.MakeType<TDataExprType>(EDataSlot::Bool);
401-
} else if (name == "int2") {
402-
dataType = ctx.MakeType<TDataExprType>(EDataSlot::Int16);
403-
} else if (name == "int4") {
404-
dataType = ctx.MakeType<TDataExprType>(EDataSlot::Int32);
405-
} else if (name == "int8") {
406-
dataType = ctx.MakeType<TDataExprType>(EDataSlot::Int64);
407-
} else if (name == "float4") {
408-
dataType = ctx.MakeType<TDataExprType>(EDataSlot::Float);
409-
} else if (name == "float8") {
410-
dataType = ctx.MakeType<TDataExprType>(EDataSlot::Double);
411-
} else if (name == "text" || name == "varchar" || name == "cstring") {
412-
dataType = ctx.MakeType<TDataExprType>(EDataSlot::Utf8);
413-
} else if (name == "bytea") {
414-
dataType = ctx.MakeType<TDataExprType>(EDataSlot::String);
415-
} else if (name == "unknown") {
416-
return ctx.MakeType<TNullExprType>();
417-
} else if (name == "date") {
418-
dataType = ctx.MakeType<TDataExprType>(EDataSlot::Date32);
419-
} else if (name == "timestamp") {
420-
dataType = ctx.MakeType<TDataExprType>(EDataSlot::Timestamp64);
421-
} else if (name == "uuid") {
422-
dataType = ctx.MakeType<TDataExprType>(EDataSlot::Uuid);
423-
} else {
397+
auto res = ConvertFromPgType(type->Cast<TPgExprType>()->GetId());
398+
if (!res) {
424399
ctx.AddError(TIssue(ctx.GetPosition(pos),
425-
TStringBuilder() << "Unsupported type: " << name));
400+
TStringBuilder() << "Unsupported type: " << *type));
426401
return nullptr;
427402
}
428403

404+
auto dataType = ctx.MakeType<TDataExprType>(*res);
429405
return ctx.MakeType<TOptionalExprType>(dataType);
430406
}
431407

@@ -464,81 +440,8 @@ const TTypeAnnotationNode* ToPgImpl(TPositionHandle pos, const TTypeAnnotationNo
464440
return nullptr;
465441
}
466442

467-
TString pgType;
468-
switch (dataType->GetSlot()) {
469-
case NUdf::EDataSlot::Bool:
470-
pgType = "bool";
471-
break;
472-
case NUdf::EDataSlot::Int16:
473-
case NUdf::EDataSlot::Int8:
474-
case NUdf::EDataSlot::Uint8:
475-
pgType = "int2";
476-
break;
477-
case NUdf::EDataSlot::Int32:
478-
case NUdf::EDataSlot::Uint16:
479-
pgType = "int4";
480-
break;
481-
case NUdf::EDataSlot::Int64:
482-
case NUdf::EDataSlot::Uint32:
483-
pgType = "int8";
484-
break;
485-
case NUdf::EDataSlot::Uint64:
486-
case NUdf::EDataSlot::Decimal:
487-
case NUdf::EDataSlot::DyNumber:
488-
pgType = "numeric";
489-
break;
490-
case NUdf::EDataSlot::Float:
491-
pgType = "float4";
492-
break;
493-
case NUdf::EDataSlot::Double:
494-
pgType = "float8";
495-
break;
496-
case NUdf::EDataSlot::String:
497-
case NUdf::EDataSlot::Yson:
498-
pgType = "bytea";
499-
break;
500-
case NUdf::EDataSlot::Utf8:
501-
case NUdf::EDataSlot::TzDate:
502-
case NUdf::EDataSlot::TzDatetime:
503-
case NUdf::EDataSlot::TzTimestamp:
504-
pgType = "text";
505-
break;
506-
case NUdf::EDataSlot::Date:
507-
case NUdf::EDataSlot::Date32:
508-
pgType = "date";
509-
break;
510-
case NUdf::EDataSlot::Datetime:
511-
case NUdf::EDataSlot::Datetime64:
512-
case NUdf::EDataSlot::Timestamp:
513-
case NUdf::EDataSlot::Timestamp64:
514-
pgType = "timestamp";
515-
break;
516-
case NUdf::EDataSlot::Interval:
517-
case NUdf::EDataSlot::Interval64:
518-
pgType = "interval";
519-
break;
520-
case NUdf::EDataSlot::Json:
521-
pgType = "json";
522-
break;
523-
case NUdf::EDataSlot::JsonDocument:
524-
pgType = "jsonb";
525-
break;
526-
case NUdf::EDataSlot::Uuid:
527-
pgType = "uuid";
528-
break;
529-
default:
530-
ctx.AddError(TIssue(ctx.GetPosition(pos),
531-
TStringBuilder() << "Unsupported type: " << dataType->GetName()));
532-
return nullptr;
533-
}
534-
535-
try {
536-
auto result = ctx.MakeType<TPgExprType>(NPg::LookupType(pgType).TypeId);
537-
return result;
538-
} catch (const yexception& e) {
539-
ctx.AddError(TIssue(ctx.GetPosition(pos), e.what()));
540-
return nullptr;
541-
}
443+
auto pgTypeId = ConvertToPgType(dataType->GetSlot());
444+
return ctx.MakeType<TPgExprType>(pgTypeId);
542445
}
543446

544447
IGraphTransformer::TStatus ToPgWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {

‎ydb/library/yql/core/yql_expr_type_annotation.cpp‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6013,14 +6013,7 @@ bool ExtractPgType(const TTypeAnnotationNode* type, ui32& pgType, bool& convertT
60136013
}
60146014

60156015
auto slot = unpacked->Cast<TDataExprType>()->GetSlot();
6016-
auto convertedTypeId = ConvertToPgType(slot);
6017-
if (!convertedTypeId) {
6018-
ctx.AddError(TIssue(ctx.GetPosition(pos),
6019-
TStringBuilder() << "Type is not compatible to PG: " << slot));
6020-
return false;
6021-
}
6022-
6023-
pgType = *convertedTypeId;
6016+
pgType = ConvertToPgType(slot);
60246017
convertToPg = true;
60256018
return true;
60266019
} else if (type->GetKind() != ETypeAnnotationKind::Pg) {

‎ydb/library/yql/parser/pg_wrapper/comp_factory.cpp‎

Lines changed: 65 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -399,8 +399,8 @@ class TPgTableContent : public TMutableComputationNode<TPgTableContent> {
399399
ApplyFillers(AllPgTablesFillers, Y_ARRAY_SIZE(AllPgTablesFillers), PgTablesFillers_);
400400
} else if (Table_ == "pg_roles") {
401401
static const std::pair<const char*, TPgRolesFiller> AllPgRolesFillers[] = {
402-
{"rolname", [](ui32 index) {
403-
return PointerDatumToPod((Datum)MakeFixedString(index == 1 ? "postgres" : *PGGetGUCSetting("ydb_user"), NAMEDATALEN));
402+
{"rolname", [](ui32 index) {
403+
return PointerDatumToPod((Datum)MakeFixedString(index == 1 ? "postgres" : *PGGetGUCSetting("ydb_user"), NAMEDATALEN));
404404
}},
405405
{"oid", [](ui32) { return ScalarDatumToPod(ObjectIdGetDatum(1)); }},
406406
{"rolbypassrls", [](ui32) { return ScalarDatumToPod(BoolGetDatum(true)); }},
@@ -489,7 +489,7 @@ class TPgTableContent : public TMutableComputationNode<TPgTableContent> {
489489
};
490490

491491
ApplyFillers(AllPgLanguageFillers, Y_ARRAY_SIZE(AllPgLanguageFillers), PgLanguageFillers_);
492-
}
492+
}
493493
} else {
494494
if (Table_ == "tables") {
495495
static const std::pair<const char*, TTablesFiller> AllTablesFillers[] = {
@@ -799,7 +799,7 @@ class TPgTableContent : public TMutableComputationNode<TPgTableContent> {
799799
});
800800

801801
for (const auto& t : tables) {
802-
const ui32 amOid = (t.Kind == NPg::ERelKind::Relation) ? btreeAmOid : 0;
802+
const ui32 amOid = (t.Kind == NPg::ERelKind::Relation) ? btreeAmOid : 0;
803803
NUdf::TUnboxedValue* items;
804804
auto row = compCtx.HolderFactory.CreateDirectArrayHolder(PgClassFillers_.size(), items);
805805
for (ui32 i = 0; i < PgClassFillers_.size(); ++i) {
@@ -1168,7 +1168,7 @@ class TPgResolvedCallBase : public TMutableComputationNode<TDerived> {
11681168
const TVector<TType*> ArgTypes;
11691169
const TStructType* StructType;
11701170
TVector<NPg::TTypeDesc> ArgDesc;
1171-
1171+
11721172
TPgArgsExprBuilder ArgsExprBuilder;
11731173
};
11741174

@@ -1312,7 +1312,7 @@ class TPgResolvedMultiCall : public TPgResolvedCallBase<TPgResolvedMultiCall> {
13121312

13131313
rsInfo.expectedDesc = BlessTupleDesc(rsInfo.expectedDesc);
13141314
}
1315-
1315+
13161316
TupleSlot = MakeSingleTupleTableSlot(rsInfo.expectedDesc, &TTSOpsMinimalTuple);
13171317
for (ui32 i = 0; i < args.size(); ++i) {
13181318
const auto& value = args[i];
@@ -1383,7 +1383,7 @@ class TPgResolvedMultiCall : public TPgResolvedCallBase<TPgResolvedMultiCall> {
13831383
FinishAndFree();
13841384
return false;
13851385
}
1386-
1386+
13871387
slot_getallattrs(TupleSlot);
13881388
if (RetTypeDesc.TypeId == RECORDOID) {
13891389
if (StructType) {
@@ -2003,7 +2003,7 @@ NUdf::TUnboxedValuePod ConvertToPgValue(NUdf::TUnboxedValuePod value, TMaybe<NUd
20032003
auto res = Timestamp2Pg(value.Get<ui64>());
20042004
return ScalarDatumToPod(res);
20052005
}
2006-
case NUdf::EDataSlot::Interval:
2006+
case NUdf::EDataSlot::Interval:
20072007
case NUdf::EDataSlot::Interval64: {
20082008
auto res = Interval2Pg(value.Get<i64>());
20092009
return PointerDatumToPod(PointerGetDatum(res));
@@ -2244,7 +2244,7 @@ class TToPg : public TMutableComputationNode<TToPg<Slot>> {
22442244

22452245
if constexpr (Slot == NUdf::EDataSlot::Decimal) {
22462246
auto decimalType = static_cast<TDataDecimalType*>(ArgType);
2247-
return PointerDatumToPod(NumericGetDatum(DecimalToPgNumeric(value,
2247+
return PointerDatumToPod(NumericGetDatum(DecimalToPgNumeric(value,
22482248
decimalType->GetParams().first, decimalType->GetParams().second)));
22492249
} else {
22502250
return ConvertToPgValue<Slot>(value);
@@ -2941,7 +2941,7 @@ struct TToPgExec {
29412941
*res = builder.Build(true);
29422942
break;
29432943
}
2944-
case NUdf::EDataSlot::Json:
2944+
case NUdf::EDataSlot::Json:
29452945
{
29462946
NUdf::TStringBlockReader<arrow::BinaryType, true> reader;
29472947
NUdf::TStringArrayBuilder<arrow::BinaryType, true> builder(NKikimr::NMiniKQL::TTypeInfoHelper(), arrow::binary(), *ctx->memory_pool(), length);
@@ -2964,7 +2964,7 @@ struct TToPgExec {
29642964
*res = builder.Build(true);
29652965
break;
29662966
}
2967-
case NUdf::EDataSlot::JsonDocument:
2967+
case NUdf::EDataSlot::JsonDocument:
29682968
{
29692969
NUdf::TStringBlockReader<arrow::BinaryType, true> reader;
29702970
NUdf::TStringArrayBuilder<arrow::BinaryType, true> builder(NKikimr::NMiniKQL::TTypeInfoHelper(), arrow::binary(), *ctx->memory_pool(), length);
@@ -2985,7 +2985,7 @@ struct TToPgExec {
29852985

29862986
*res = builder.Build(true);
29872987
break;
2988-
}
2988+
}
29892989
default:
29902990
ythrow yexception() << "Unsupported type: " << NUdf::GetDataTypeInfo(SourceDataSlot).Name;
29912991
}
@@ -4013,16 +4013,26 @@ arrow::Datum MakePgScalar(NKikimr::NMiniKQL::TPgType* type, const NUdf::TBlockIt
40134013
);
40144014
}
40154015

4016-
TMaybe<ui32> ConvertToPgType(NUdf::EDataSlot slot) {
4016+
ui32 ConvertToPgType(NUdf::EDataSlot slot) {
40174017
switch (slot) {
40184018
case NUdf::EDataSlot::Bool:
40194019
return BOOLOID;
4020+
case NUdf::EDataSlot::Int8:
4021+
return INT2OID;
4022+
case NUdf::EDataSlot::Uint8:
4023+
return INT2OID;
40204024
case NUdf::EDataSlot::Int16:
40214025
return INT2OID;
4026+
case NUdf::EDataSlot::Uint16:
4027+
return INT4OID;
40224028
case NUdf::EDataSlot::Int32:
40234029
return INT4OID;
4030+
case NUdf::EDataSlot::Uint32:
4031+
return INT8OID;
40244032
case NUdf::EDataSlot::Int64:
40254033
return INT8OID;
4034+
case NUdf::EDataSlot::Uint64:
4035+
return NUMERICOID;
40264036
case NUdf::EDataSlot::Float:
40274037
return FLOAT4OID;
40284038
case NUdf::EDataSlot::Double:
@@ -4031,8 +4041,40 @@ TMaybe<ui32> ConvertToPgType(NUdf::EDataSlot slot) {
40314041
return BYTEAOID;
40324042
case NUdf::EDataSlot::Utf8:
40334043
return TEXTOID;
4034-
default:
4035-
return Nothing();
4044+
case NUdf::EDataSlot::Yson:
4045+
return BYTEAOID;
4046+
case NUdf::EDataSlot::Json:
4047+
return JSONOID;
4048+
case NUdf::EDataSlot::Uuid:
4049+
return UUIDOID;
4050+
case NUdf::EDataSlot::Date:
4051+
return DATEOID;
4052+
case NUdf::EDataSlot::Datetime:
4053+
return TIMESTAMPOID;
4054+
case NUdf::EDataSlot::Timestamp:
4055+
return TIMESTAMPOID;
4056+
case NUdf::EDataSlot::Interval:
4057+
return INTERVALOID;
4058+
case NUdf::EDataSlot::TzDate:
4059+
return TEXTOID;
4060+
case NUdf::EDataSlot::TzDatetime:
4061+
return TEXTOID;
4062+
case NUdf::EDataSlot::TzTimestamp:
4063+
return TEXTOID;
4064+
case NUdf::EDataSlot::Decimal:
4065+
return NUMERICOID;
4066+
case NUdf::EDataSlot::DyNumber:
4067+
return NUMERICOID;
4068+
case NUdf::EDataSlot::JsonDocument:
4069+
return JSONBOID;
4070+
case NUdf::EDataSlot::Date32:
4071+
return DATEOID;
4072+
case NUdf::EDataSlot::Datetime64:
4073+
return TIMESTAMPOID;
4074+
case NUdf::EDataSlot::Timestamp64:
4075+
return TIMESTAMPOID;
4076+
case NUdf::EDataSlot::Interval64:
4077+
return INTERVALOID;
40364078
}
40374079
}
40384080

@@ -4053,7 +4095,15 @@ TMaybe<NUdf::EDataSlot> ConvertFromPgType(ui32 typeId) {
40534095
case BYTEAOID:
40544096
return NUdf::EDataSlot::String;
40554097
case TEXTOID:
4098+
case VARCHAROID:
4099+
case CSTRINGOID:
40564100
return NUdf::EDataSlot::Utf8;
4101+
case DATEOID:
4102+
return NUdf::EDataSlot::Date32;
4103+
case TIMESTAMPOID:
4104+
return NUdf::EDataSlot::Timestamp64;
4105+
case UUIDOID:
4106+
return NUdf::EDataSlot::Uuid;
40574107
}
40584108

40594109
return Nothing();

‎ydb/library/yql/parser/pg_wrapper/interface/utils.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
namespace NYql {
99

10-
TMaybe<ui32> ConvertToPgType(NKikimr::NUdf::EDataSlot slot);
10+
ui32 ConvertToPgType(NKikimr::NUdf::EDataSlot slot);
1111
TMaybe<NKikimr::NUdf::EDataSlot> ConvertFromPgType(ui32 typeId);
1212

1313
bool ParsePgIntervalModifier(const TString& str, i32& ret);

‎ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -289,9 +289,9 @@ TColumnConverter BuildPgColumnConverter(const std::shared_ptr<arrow::DataType>&
289289
return {};
290290
}
291291

292-
TMaybe<ui32> ConvertToPgType(NKikimr::NUdf::EDataSlot slot) {
292+
ui32 ConvertToPgType(NKikimr::NUdf::EDataSlot slot) {
293293
Y_UNUSED(slot);
294-
return Nothing();
294+
throw yexception() << "PG types are not supported";
295295
}
296296

297297
TMaybe<NKikimr::NUdf::EDataSlot> ConvertFromPgType(ui32 typeId) {

‎ydb/library/yql/tests/sql/dq_file/part4/canondata/result.json‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1932,6 +1932,28 @@
19321932
}
19331933
],
19341934
"test.test[order_by-sort_with_take_limit--Results]": [],
1935+
"test.test[pg-all_data--Analyze]": [
1936+
{
1937+
"checksum": "c5803b9f3080a48130c30f8602733de3",
1938+
"size": 4575,
1939+
"uri": "https://{canondata_backend}/1599023/6add8cb499cc3b1dca20f22c9b17ae29fbfe727d/resource.tar.gz#test.test_pg-all_data--Analyze_/plan.txt"
1940+
}
1941+
],
1942+
"test.test[pg-all_data--Debug]": [
1943+
{
1944+
"checksum": "8fabd9ef07df1dc7f87bf4f852b9e0bf",
1945+
"size": 5441,
1946+
"uri": "https://{canondata_backend}/1599023/6add8cb499cc3b1dca20f22c9b17ae29fbfe727d/resource.tar.gz#test.test_pg-all_data--Debug_/opt.yql_patched"
1947+
}
1948+
],
1949+
"test.test[pg-all_data--Plan]": [
1950+
{
1951+
"checksum": "c5803b9f3080a48130c30f8602733de3",
1952+
"size": 4575,
1953+
"uri": "https://{canondata_backend}/1599023/6add8cb499cc3b1dca20f22c9b17ae29fbfe727d/resource.tar.gz#test.test_pg-all_data--Plan_/plan.txt"
1954+
}
1955+
],
1956+
"test.test[pg-all_data--Results]": [],
19351957
"test.test[pg-drop_table--Analyze]": [
19361958
{
19371959
"checksum": "5ba98022094d906a6acb7cf1b61f14ed",

0 commit comments

Comments
 (0)