Skip to content

Commit 06791fb

Browse files
nepalmaximyurchuk
nepal
authored and
maximyurchuk
committed
Merge PR #10831, #11068, #11075, #11152
#11152 - Allow to choose normal or aggreation PG function #11075 - Handle invalid base #11068 - Allow more postgis functions #10831 - Views: if exists / if not exists for DDL commit_hash:0ebf35e45ac6de147c9000440ca25237db061d2e
1 parent 4e4600b commit 06791fb

File tree

17 files changed

+260
-53
lines changed

17 files changed

+260
-53
lines changed

‎yql/essentials/parser/pg_catalog/catalog.cpp‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2362,6 +2362,24 @@ void EnumProc(std::function<void(ui32, const TProcDesc&)> f) {
23622362
}
23632363
}
23642364

2365+
bool HasProc(const TString& name, EProcKind kind) {
2366+
const auto& catalog = TCatalog::Instance();
2367+
auto procIdPtr = catalog.State->ProcByName.FindPtr(to_lower(name));
2368+
if (!procIdPtr) {
2369+
return false;
2370+
}
2371+
2372+
for (const auto& id : *procIdPtr) {
2373+
const auto& d = catalog.State->Procs.FindPtr(id);
2374+
Y_ENSURE(d);
2375+
if (d->Kind == kind) {
2376+
return true;
2377+
}
2378+
}
2379+
2380+
return false;
2381+
}
2382+
23652383
bool HasReturnSetProc(const TString& name) {
23662384
const auto& catalog = TCatalog::Instance();
23672385
auto procIdPtr = catalog.State->ProcByName.FindPtr(to_lower(name));

‎yql/essentials/parser/pg_catalog/catalog.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,7 @@ const TProcDesc& LookupProc(ui32 procId);
283283
std::variant<const TProcDesc*, const TTypeDesc*> LookupProcWithCasts(const TString& name, const TVector<ui32>& argTypeIds);
284284
bool HasReturnSetProc(const TString& name);
285285
void EnumProc(std::function<void(ui32, const TProcDesc&)> f);
286+
bool HasProc(const TString& name, EProcKind kind);
286287

287288
bool HasType(const TString& name);
288289
bool HasType(ui32 typeId);

‎yql/essentials/parser/pg_catalog/safe_procs.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@
3939

4040
// postgis
4141
"postgis_lib_version",
42+
"st_interiorringn",
43+
"st_collectionhomogenize",
44+
"st_makeline",
4245

4346
//dbeaver
4447
"pg_get_expr",

‎yql/essentials/sql/v1/SQLv1.g.in‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -601,12 +601,12 @@ alter_external_data_source_action:
601601

602602
drop_external_data_source_stmt: DROP EXTERNAL DATA SOURCE (IF EXISTS)? object_ref;
603603

604-
create_view_stmt: CREATE VIEW object_ref
604+
create_view_stmt: CREATE VIEW (IF NOT EXISTS)? object_ref
605605
create_object_features?
606606
AS select_stmt
607607
;
608608

609-
drop_view_stmt: DROP VIEW object_ref;
609+
drop_view_stmt: DROP VIEW (IF EXISTS)? object_ref;
610610

611611
upsert_object_stmt: UPSERT OBJECT object_ref
612612
LPAREN TYPE object_type_ref RPAREN

‎yql/essentials/sql/v1/SQLv1Antlr4.g.in‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -600,12 +600,12 @@ alter_external_data_source_action:
600600

601601
drop_external_data_source_stmt: DROP EXTERNAL DATA SOURCE (IF EXISTS)? object_ref;
602602

603-
create_view_stmt: CREATE VIEW object_ref
603+
create_view_stmt: CREATE VIEW (IF NOT EXISTS)? object_ref
604604
create_object_features?
605605
AS select_stmt
606606
;
607607

608-
drop_view_stmt: DROP VIEW object_ref;
608+
drop_view_stmt: DROP VIEW (IF EXISTS)? object_ref;
609609

610610
upsert_object_stmt: UPSERT OBJECT object_ref
611611
LPAREN TYPE object_type_ref RPAREN

‎yql/essentials/sql/v1/builtin.cpp‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3421,15 +3421,39 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec
34213421
}
34223422
} else if (ns == "datetime2" && (name == "Parse")) {
34233423
return BuildUdf(ctx, pos, nameSpace, name, args);
3424-
} else if (ns == "pg") {
3425-
const bool isAggregateFunc = NYql::NPg::HasAggregation(name, NYql::NPg::EAggKind::Normal);
3424+
} else if (ns == "pg" || ns == "pgagg" || ns == "pgproc") {
3425+
bool isAggregateFunc = NYql::NPg::HasAggregation(name, NYql::NPg::EAggKind::Normal);
3426+
bool isNormalFunc = NYql::NPg::HasProc(name, NYql::NPg::EProcKind::Function);
3427+
if (!isAggregateFunc && !isNormalFunc) {
3428+
return new TInvalidBuiltin(pos, TStringBuilder() << "Unknown function: " << name);
3429+
}
3430+
3431+
if (isAggregateFunc && isNormalFunc) {
3432+
if (ns == "pg") {
3433+
return new TInvalidBuiltin(pos, TStringBuilder() << "Ambigious function: " << name << ", use either PgAgg:: or PgProc:: namespace");
3434+
} else if (ns == "pgagg") {
3435+
isNormalFunc = false;
3436+
} else {
3437+
isAggregateFunc = false;
3438+
}
3439+
}
3440+
3441+
if (isAggregateFunc && ns == "pgproc") {
3442+
return new TInvalidBuiltin(pos, TStringBuilder() << "Invalid namespace for aggregation function: " << name << ", use either Pg:: or PgAgg:: namespace");
3443+
}
3444+
3445+
if (isNormalFunc && ns == "pgagg") {
3446+
return new TInvalidBuiltin(pos, TStringBuilder() << "Invalid namespace for normal function: " << name << ", use either Pg:: or PgProc:: namespace");
3447+
}
3448+
34263449
if (isAggregateFunc) {
34273450
if (aggMode == EAggregateMode::Distinct) {
34283451
return new TInvalidBuiltin(pos, "Distinct is not supported yet for PG aggregation ");
34293452
}
34303453

34313454
return BuildAggrFuncFactoryCallback(name, "", EAggrFuncTypeCallback::PG)(pos, args, aggMode, false);
34323455
} else {
3456+
YQL_ENSURE(isNormalFunc);
34333457
TVector<TNodePtr> pgCallArgs;
34343458
pgCallArgs.push_back(BuildLiteralRawString(pos, name));
34353459
pgCallArgs.insert(pgCallArgs.end(), args.begin(), args.end());

‎yql/essentials/sql/v1/format/sql_format_ut.h‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1538,19 +1538,30 @@ Y_UNIT_TEST(ObfuscatePragma) {
15381538
}
15391539

15401540
Y_UNIT_TEST(CreateView) {
1541-
TCases cases = {
1542-
{"creAte vIEw TheView wiTh (security_invoker = trUE) As SELect 1",
1543-
"CREATE VIEW TheView WITH (security_invoker = TRUE) AS\nSELECT\n\t1;\n"},
1541+
TCases cases = {{
1542+
"creAte vIEw TheView As SELect 1",
1543+
"CREATE VIEW TheView AS\nSELECT\n\t1;\n"
1544+
}, {
1545+
"creAte vIEw If Not ExIsTs TheView As SELect 1",
1546+
"CREATE VIEW IF NOT EXISTS TheView AS\nSELECT\n\t1;\n"
1547+
}, {
1548+
"creAte vIEw TheView wiTh (option = tRuE) As SELect 1",
1549+
"CREATE VIEW TheView WITH (option = TRUE) AS\nSELECT\n\t1;\n"
1550+
}
15441551
};
15451552

15461553
TSetup setup;
15471554
setup.Run(cases);
15481555
}
15491556

15501557
Y_UNIT_TEST(DropView) {
1551-
TCases cases = {
1552-
{"dRop viEW theVIEW",
1553-
"DROP VIEW theVIEW;\n"},
1558+
TCases cases = {{
1559+
"dRop viEW theVIEW",
1560+
"DROP VIEW theVIEW;\n"
1561+
}, {
1562+
"dRop viEW iF EXistS theVIEW",
1563+
"DROP VIEW IF EXISTS theVIEW;\n"
1564+
}
15541565
};
15551566

15561567
TSetup setup;

‎yql/essentials/sql/v1/sql_query.cpp‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1228,60 +1228,64 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
12281228
break;
12291229
}
12301230
case TRule_sql_stmt_core::kAltSqlStmtCore42: {
1231-
// create_view_stmt: CREATE VIEW name WITH (k = v, ...) AS select_stmt;
1231+
// create_view_stmt: CREATE VIEW (IF NOT EXISTS)? name (WITH (k = v, ...))? AS select_stmt;
12321232
auto& node = core.GetAlt_sql_stmt_core42().GetRule_create_view_stmt1();
12331233
TObjectOperatorContext context(Ctx.Scoped);
1234-
if (node.GetRule_object_ref3().HasBlock1()) {
1235-
if (!ClusterExpr(node.GetRule_object_ref3().GetBlock1().GetRule_cluster_expr1(),
1234+
if (node.GetRule_object_ref4().HasBlock1()) {
1235+
if (!ClusterExpr(node.GetRule_object_ref4().GetBlock1().GetRule_cluster_expr1(),
12361236
false,
12371237
context.ServiceId,
12381238
context.Cluster)) {
12391239
return false;
12401240
}
12411241
}
12421242

1243+
const bool existingOk = node.HasBlock3();
1244+
12431245
std::map<TString, TDeferredAtom> features;
1244-
if (node.HasBlock4()) {
1245-
if (!ParseObjectFeatures(features, node.GetBlock4().GetRule_create_object_features1().GetRule_object_features2())) {
1246+
if (node.HasBlock5()) {
1247+
if (!ParseObjectFeatures(features, node.GetBlock5().GetRule_create_object_features1().GetRule_object_features2())) {
12461248
return false;
12471249
}
12481250
}
1249-
if (!ParseViewQuery(features, node.GetRule_select_stmt6())) {
1251+
if (!ParseViewQuery(features, node.GetRule_select_stmt7())) {
12501252
return false;
12511253
}
12521254

1253-
const TString objectId = Id(node.GetRule_object_ref3().GetRule_id_or_at2(), *this).second;
1255+
const TString objectId = Id(node.GetRule_object_ref4().GetRule_id_or_at2(), *this).second;
12541256
constexpr const char* TypeId = "VIEW";
12551257
AddStatementToBlocks(blocks,
12561258
BuildCreateObjectOperation(Ctx.Pos(),
12571259
BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId),
12581260
TypeId,
1259-
false,
1261+
existingOk,
12601262
false,
12611263
std::move(features),
12621264
context));
12631265
break;
12641266
}
12651267
case TRule_sql_stmt_core::kAltSqlStmtCore43: {
1266-
// drop_view_stmt: DROP VIEW name;
1268+
// drop_view_stmt: DROP VIEW (IF EXISTS)? name;
12671269
auto& node = core.GetAlt_sql_stmt_core43().GetRule_drop_view_stmt1();
12681270
TObjectOperatorContext context(Ctx.Scoped);
1269-
if (node.GetRule_object_ref3().HasBlock1()) {
1270-
if (!ClusterExpr(node.GetRule_object_ref3().GetBlock1().GetRule_cluster_expr1(),
1271+
if (node.GetRule_object_ref4().HasBlock1()) {
1272+
if (!ClusterExpr(node.GetRule_object_ref4().GetBlock1().GetRule_cluster_expr1(),
12711273
false,
12721274
context.ServiceId,
12731275
context.Cluster)) {
12741276
return false;
12751277
}
12761278
}
12771279

1278-
const TString objectId = Id(node.GetRule_object_ref3().GetRule_id_or_at2(), *this).second;
1280+
const bool missingOk = node.HasBlock3();
1281+
1282+
const TString objectId = Id(node.GetRule_object_ref4().GetRule_id_or_at2(), *this).second;
12791283
constexpr const char* TypeId = "VIEW";
12801284
AddStatementToBlocks(blocks,
12811285
BuildDropObjectOperation(Ctx.Pos(),
12821286
BuildTablePath(Ctx.GetPrefixPath(context.ServiceId, context.Cluster), objectId),
12831287
TypeId,
1284-
false,
1288+
missingOk,
12851289
{},
12861290
context));
12871291
break;

‎yql/essentials/sql/v1/sql_ut.cpp‎

Lines changed: 58 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2533,8 +2533,8 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
25332533
const auto result = SqlToYql(R"(USE plato;
25342534
CREATE TABLE table (
25352535
pk INT32 NOT NULL,
2536-
col String,
2537-
INDEX idx GLOBAL USING vector_kmeans_tree
2536+
col String,
2537+
INDEX idx GLOBAL USING vector_kmeans_tree
25382538
ON (col) COVER (col)
25392539
WITH (distance=cosine, vector_type=float, vector_dimension=1024,),
25402540
PRIMARY KEY (pk))
@@ -2543,11 +2543,11 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
25432543
}
25442544

25452545
Y_UNIT_TEST(AlterTableAddIndexVector) {
2546-
const auto result = SqlToYql(R"(USE plato;
2547-
ALTER TABLE table ADD INDEX idx
2548-
GLOBAL USING vector_kmeans_tree
2546+
const auto result = SqlToYql(R"(USE plato;
2547+
ALTER TABLE table ADD INDEX idx
2548+
GLOBAL USING vector_kmeans_tree
25492549
ON (col) COVER (col)
2550-
WITH (distance=cosine, vector_type="float", vector_dimension=1024)
2550+
WITH (distance=cosine, vector_type="float", vector_dimension=1024)
25512551
)");
25522552
UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString());
25532553
}
@@ -2558,11 +2558,11 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
25582558
}
25592559

25602560
Y_UNIT_TEST(AlterTableAddIndexMissedParameter) {
2561-
ExpectFailWithError(R"(USE plato;
2562-
ALTER TABLE table ADD INDEX idx
2563-
GLOBAL USING vector_kmeans_tree
2561+
ExpectFailWithError(R"(USE plato;
2562+
ALTER TABLE table ADD INDEX idx
2563+
GLOBAL USING vector_kmeans_tree
25642564
ON (col)
2565-
WITH (distance=cosine, vector_type=float)
2565+
WITH (distance=cosine, vector_type=float)
25662566
)",
25672567
"<main>:5:52: Error: vector_dimension should be set\n");
25682568
}
@@ -2790,7 +2790,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
27902790
auto req = Sprintf(reqTpl, key.c_str(), value.c_str());
27912791
auto res = SqlToYql(req);
27922792
UNIT_ASSERT(res.Root);
2793-
2793+
27942794
TVerifyLineFunc verifyLine = [&key, &value](const TString& word, const TString& line) {
27952795
if (word == "Write") {
27962796
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("MyReplication"));
@@ -6716,6 +6716,28 @@ Y_UNIT_TEST_SUITE(TViewSyntaxTest) {
67166716
UNIT_ASSERT_C(res.Root, res.Issues.ToString());
67176717
}
67186718

6719+
Y_UNIT_TEST(CreateViewIfNotExists) {
6720+
constexpr const char* name = "TheView";
6721+
NYql::TAstParseResult res = SqlToYql(std::format(R"(
6722+
USE plato;
6723+
CREATE VIEW IF NOT EXISTS {} WITH (security_invoker = TRUE) AS SELECT 1;
6724+
)", name
6725+
));
6726+
UNIT_ASSERT_C(res.Root, res.Issues.ToString());
6727+
6728+
TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) {
6729+
if (word == "Write!") {
6730+
UNIT_ASSERT_STRING_CONTAINS(line, name);
6731+
UNIT_ASSERT_STRING_CONTAINS(line, "createObjectIfNotExists");
6732+
}
6733+
};
6734+
6735+
TWordCountHive elementStat = { {"Write!"} };
6736+
VerifyProgram(res, elementStat, verifyLine);
6737+
6738+
UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1);
6739+
}
6740+
67196741
Y_UNIT_TEST(CreateViewFromTable) {
67206742
constexpr const char* path = "/PathPrefix/TheView";
67216743
constexpr const char* query = R"(
@@ -6795,6 +6817,28 @@ Y_UNIT_TEST_SUITE(TViewSyntaxTest) {
67956817
UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1);
67966818
}
67976819

6820+
Y_UNIT_TEST(DropViewIfExists) {
6821+
constexpr const char* name = "TheView";
6822+
NYql::TAstParseResult res = SqlToYql(std::format(R"(
6823+
USE plato;
6824+
DROP VIEW IF EXISTS {};
6825+
)", name
6826+
));
6827+
UNIT_ASSERT_C(res.Root, res.Issues.ToString());
6828+
6829+
TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) {
6830+
if (word == "Write!") {
6831+
UNIT_ASSERT_STRING_CONTAINS(line, name);
6832+
UNIT_ASSERT_STRING_CONTAINS(line, "dropObjectIfExists");
6833+
}
6834+
};
6835+
6836+
TWordCountHive elementStat = { {"Write!"} };
6837+
VerifyProgram(res, elementStat, verifyLine);
6838+
6839+
UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1);
6840+
}
6841+
67986842
Y_UNIT_TEST(CreateViewWithTablePrefix) {
67996843
NYql::TAstParseResult res = SqlToYql(R"(
68006844
USE plato;
@@ -6838,7 +6882,7 @@ Y_UNIT_TEST_SUITE(TViewSyntaxTest) {
68386882

68396883
UNIT_ASSERT_VALUES_EQUAL(elementStat["Write!"], 1);
68406884
}
6841-
6885+
68426886
Y_UNIT_TEST(YtAlternativeSchemaSyntax) {
68436887
NYql::TAstParseResult res = SqlToYql(R"(
68446888
SELECT * FROM plato.Input WITH schema(y Int32, x String not null);
@@ -6907,7 +6951,7 @@ Y_UNIT_TEST_SUITE(CompactNamedExprs) {
69076951
pragma CompactNamedExprs;
69086952
pragma ValidateUnusedExprs;
69096953
6910-
define subquery $x() as
6954+
define subquery $x() as
69116955
select count(1, 2);
69126956
end define;
69136957
select 1;
@@ -6930,7 +6974,7 @@ Y_UNIT_TEST_SUITE(CompactNamedExprs) {
69306974
pragma CompactNamedExprs;
69316975
pragma DisableValidateUnusedExprs;
69326976
6933-
define subquery $x() as
6977+
define subquery $x() as
69346978
select count(1, 2);
69356979
end define;
69366980
select 1;

0 commit comments

Comments
 (0)