Skip to content

YQL-17264 static part of tables #1327

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 26, 2024
Merged

YQL-17264 static part of tables #1327

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
485 changes: 278 additions & 207 deletions ydb/library/yql/parser/pg_wrapper/comp_factory.cpp
View file
Open in desktop

Large diffs are not rendered by default.

78 changes: 49 additions & 29 deletions ydb/library/yql/providers/pg/provider/yql_pg_datasource_type_ann.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ class TPgDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase {
cluster = input->Child(TNode::idx_Cluster)->Content();
}

if (cluster != "pg_catalog") {
if (cluster != "pg_catalog" && cluster != "information_schema") {
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Unexpected cluster: " << cluster));
return TStatus::Error;
}
Expand Down Expand Up @@ -107,35 +107,45 @@ class TPgDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase {

auto tableName = input->Child(TNode::idx_Table)->Content();
TVector<const TItemExprType*> items;
if (tableName == "pg_type") {
FillPgTypeSchema(items, ctx);
} else if (tableName == "pg_database") {
FillPgDatabaseSchema(items, ctx);
} else if (tableName == "pg_tablespace") {
FillPgTablespaceSchema(items, ctx);
} else if (tableName == "pg_shdescription") {
FillPgShDescriptionSchema(items, ctx);
} else if (tableName == "pg_trigger") {
FillPgTriggerSchema(items, ctx);
} else if (tableName == "pg_locks") {
FillPgLocksSchema(items, ctx);
} else if (tableName == "pg_stat_gssapi") {
FillPgStatGssapiSchema(items, ctx);
} else if (tableName == "pg_inherits") {
FillPgInheritsSchema(items, ctx);
} else if (tableName == "pg_stat_activity") {
FillPgStatActivitySchema(items, ctx);
} else if (tableName == "pg_timezone_names") {
FillPgTimezoneNamesSchema(items, ctx);
} else if (tableName == "pg_timezone_abbrevs") {
FillPgTimezoneAbbrevsSchema(items, ctx);
} else if (tableName == "pg_namespace") {
FillPgNamespaceSchema(items, ctx);
} else if (tableName == "pg_description") {
FillPgDescriptionSchema(items, ctx);
} else if (tableName == "pg_am") {
FillPgAmSchema(items, ctx);
if (cluster == "pg_catalog") {
if (tableName == "pg_type") {
FillPgTypeSchema(items, ctx);
} else if (tableName == "pg_database") {
FillPgDatabaseSchema(items, ctx);
} else if (tableName == "pg_tablespace") {
FillPgTablespaceSchema(items, ctx);
} else if (tableName == "pg_shdescription") {
FillPgShDescriptionSchema(items, ctx);
} else if (tableName == "pg_trigger") {
FillPgTriggerSchema(items, ctx);
} else if (tableName == "pg_locks") {
FillPgLocksSchema(items, ctx);
} else if (tableName == "pg_stat_gssapi") {
FillPgStatGssapiSchema(items, ctx);
} else if (tableName == "pg_inherits") {
FillPgInheritsSchema(items, ctx);
} else if (tableName == "pg_stat_activity") {
FillPgStatActivitySchema(items, ctx);
} else if (tableName == "pg_timezone_names") {
FillPgTimezoneNamesSchema(items, ctx);
} else if (tableName == "pg_timezone_abbrevs") {
FillPgTimezoneAbbrevsSchema(items, ctx);
} else if (tableName == "pg_namespace") {
FillPgNamespaceSchema(items, ctx);
} else if (tableName == "pg_description") {
FillPgDescriptionSchema(items, ctx);
} else if (tableName == "pg_am") {
FillPgAmSchema(items, ctx);
} else if (tableName == "pg_tables") {
FillPgTablesSchema(items, ctx);
}
} else {
if (tableName == "tables") {
FillTablesSchema(items, ctx);
}
}

if (items.empty()) {
ctx.AddError(TIssue(ctx.GetPosition(input->Child(TPgReadTable::idx_Table)->Pos()), TStringBuilder() << "Unsupported table: " << tableName));
return TStatus::Error;
}
Expand Down Expand Up @@ -259,6 +269,16 @@ class TPgDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase {
AddColumn(items, ctx, "amtype", "char");
}

void FillPgTablesSchema(TVector<const TItemExprType*>& items, TExprContext& ctx) {
AddColumn(items, ctx, "schemaname", "name");
AddColumn(items, ctx, "tablename", "name");
}

void FillTablesSchema(TVector<const TItemExprType*>& items, TExprContext& ctx) {
AddColumn(items, ctx, "table_schema", "name");
AddColumn(items, ctx, "table_name", "name");
}

void AddColumn(TVector<const TItemExprType*>& items, TExprContext& ctx, const TString& name, const TString& type) {
items.push_back(ctx.MakeType<TItemExprType>(name, ctx.MakeType<TPgExprType>(NPg::LookupType(type).TypeId)));
}
Expand Down
22 changes: 22 additions & 0 deletions ydb/library/yql/tests/sql/dq_file/part14/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2435,6 +2435,28 @@
}
],
"test.test[pg_catalog-pg_stat_activity-default.txt-Results]": [],
"test.test[pg_catalog-pg_tables-default.txt-Analyze]": [
{
"checksum": "c1f2d837c3623c81dd596a9877913fb8",
"size": 948,
"uri": "https://{canondata_backend}/1942415/8d010130e1284b2e3b1f4a934fc3768cc8409e69/resource.tar.gz#test.test_pg_catalog-pg_tables-default.txt-Analyze_/plan.txt"
}
],
"test.test[pg_catalog-pg_tables-default.txt-Debug]": [
{
"checksum": "52297e9a1b7af4cacb9a5bf01b89b20e",
"size": 548,
"uri": "https://{canondata_backend}/1942415/8d010130e1284b2e3b1f4a934fc3768cc8409e69/resource.tar.gz#test.test_pg_catalog-pg_tables-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg_catalog-pg_tables-default.txt-Plan]": [
{
"checksum": "c1f2d837c3623c81dd596a9877913fb8",
"size": 948,
"uri": "https://{canondata_backend}/1942415/8d010130e1284b2e3b1f4a934fc3768cc8409e69/resource.tar.gz#test.test_pg_catalog-pg_tables-default.txt-Plan_/plan.txt"
}
],
"test.test[pg_catalog-pg_tables-default.txt-Results]": [],
"test.test[sampling-mapjoin_right_sample-default.txt-Analyze]": [
{
"checksum": "87b97895afd9ac6a3350f69a77811ed1",
Expand Down
22 changes: 22 additions & 0 deletions ydb/library/yql/tests/sql/dq_file/part3/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1962,6 +1962,28 @@
}
],
"test.test[pg_catalog-pg_am_pg_syntax-default.txt-Results]": [],
"test.test[pg_catalog-tables-default.txt-Analyze]": [
{
"checksum": "c1f2d837c3623c81dd596a9877913fb8",
"size": 948,
"uri": "https://{canondata_backend}/1923547/63a9bf11f98a47ebc6a355858fc4c8179b67ce82/resource.tar.gz#test.test_pg_catalog-tables-default.txt-Analyze_/plan.txt"
}
],
"test.test[pg_catalog-tables-default.txt-Debug]": [
{
"checksum": "2e8892e4dc2cd80714860bf3b4264747",
"size": 562,
"uri": "https://{canondata_backend}/1923547/63a9bf11f98a47ebc6a355858fc4c8179b67ce82/resource.tar.gz#test.test_pg_catalog-tables-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[pg_catalog-tables-default.txt-Plan]": [
{
"checksum": "c1f2d837c3623c81dd596a9877913fb8",
"size": 948,
"uri": "https://{canondata_backend}/1923547/63a9bf11f98a47ebc6a355858fc4c8179b67ce82/resource.tar.gz#test.test_pg_catalog-tables-default.txt-Plan_/plan.txt"
}
],
"test.test[pg_catalog-tables-default.txt-Results]": [],
"test.test[produce-process_row_and_columns-default.txt-Analyze]": [
{
"checksum": "ebe34f5d2b86ce7954a161dc02a783aa",
Expand Down
14 changes: 14 additions & 0 deletions ydb/library/yql/tests/sql/sql2yql/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13320,6 +13320,13 @@
"uri": "https://{canondata_backend}/1773845/118c740ff9ea824c1b2998c5bad993b2a2276f6e/resource.tar.gz#test_sql2yql.test_pg_catalog-pg_stat_gssapi_/sql.yql"
}
],
"test_sql2yql.test[pg_catalog-pg_tables]": [
{
"checksum": "83d08986e200ccb6828e343083d43e97",
"size": 853,
"uri": "https://{canondata_backend}/1942100/35599942a1c753fdf2120a615bed33fc024774bc/resource.tar.gz#test_sql2yql.test_pg_catalog-pg_tables_/sql.yql"
}
],
"test_sql2yql.test[pg_catalog-pg_tablespace]": [
{
"checksum": "755687f11614cdbe84381134ddba275c",
Expand Down Expand Up @@ -13362,6 +13369,13 @@
"uri": "https://{canondata_backend}/1942173/cb2fdcf9620011c608e9c76afbfe1a17bdd9d6f4/resource.tar.gz#test_sql2yql.test_pg_catalog-pg_type_syntax_yql_/sql.yql"
}
],
"test_sql2yql.test[pg_catalog-tables]": [
{
"checksum": "fb7f166f5cfb2889f9271ec26d395289",
"size": 864,
"uri": "https://{canondata_backend}/1942100/35599942a1c753fdf2120a615bed33fc024774bc/resource.tar.gz#test_sql2yql.test_pg_catalog-tables_/sql.yql"
}
],
"test_sql2yql.test[pragma-classic_division]": [
{
"checksum": "3e4104661f310517756ba3c530ebd08e",
Expand Down
7 changes: 7 additions & 0 deletions ydb/library/yql/tests/sql/suites/pg_catalog/pg_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
--!syntax_pg
select
schemaname,
tablename
from pg_catalog.pg_tables
order by schemaname,tablename;

7 changes: 7 additions & 0 deletions ydb/library/yql/tests/sql/suites/pg_catalog/tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
--!syntax_pg
select
table_schema,
table_name
from information_schema.tables
order by table_schema, table_name;

21 changes: 21 additions & 0 deletions ydb/library/yql/tests/sql/yt_native_file/part14/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2370,6 +2370,27 @@
"uri": "https://{canondata_backend}/1880306/1ee788a2569716571a36649ae2ceeb626ced8876/resource.tar.gz#test.test_pg_catalog-pg_stat_activity-default.txt-Results_/results.txt"
}
],
"test.test[pg_catalog-pg_tables-default.txt-Debug]": [
{
"checksum": "85f7ad11d93163d11c5a12a8df3dec7f",
"size": 489,
"uri": "https://{canondata_backend}/1600758/bdc527da4970a74ab81bbc76ba66f5dbb6340187/resource.tar.gz#test.test_pg_catalog-pg_tables-default.txt-Debug_/opt.yql"
}
],
"test.test[pg_catalog-pg_tables-default.txt-Plan]": [
{
"checksum": "c1f2d837c3623c81dd596a9877913fb8",
"size": 948,
"uri": "https://{canondata_backend}/1600758/bdc527da4970a74ab81bbc76ba66f5dbb6340187/resource.tar.gz#test.test_pg_catalog-pg_tables-default.txt-Plan_/plan.txt"
}
],
"test.test[pg_catalog-pg_tables-default.txt-Results]": [
{
"checksum": "40a0cc11ca3cade6503a95d7d62e609c",
"size": 2450,
"uri": "https://{canondata_backend}/1600758/bdc527da4970a74ab81bbc76ba66f5dbb6340187/resource.tar.gz#test.test_pg_catalog-pg_tables-default.txt-Results_/results.txt"
}
],
"test.test[produce-process_rows_sorted_desc_multi_out--Debug]": [
{
"checksum": "202fbb809a2ada2cd5905af79fdf20b0",
Expand Down
21 changes: 21 additions & 0 deletions ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1655,6 +1655,27 @@
"uri": "https://{canondata_backend}/1597364/7422967ce2af0a6bf5112b931f1ddb2e9b629e98/resource.tar.gz#test.test_pg_catalog-pg_am_pg_syntax-default.txt-Results_/results.txt"
}
],
"test.test[pg_catalog-tables-default.txt-Debug]": [
{
"checksum": "2efbb7e2f9c464af418ffb3775c9fd13",
"size": 501,
"uri": "https://{canondata_backend}/1942100/3f6203e58cb404b40c036990713353e16c2dde44/resource.tar.gz#test.test_pg_catalog-tables-default.txt-Debug_/opt.yql"
}
],
"test.test[pg_catalog-tables-default.txt-Plan]": [
{
"checksum": "c1f2d837c3623c81dd596a9877913fb8",
"size": 948,
"uri": "https://{canondata_backend}/1942100/3f6203e58cb404b40c036990713353e16c2dde44/resource.tar.gz#test.test_pg_catalog-tables-default.txt-Plan_/plan.txt"
}
],
"test.test[pg_catalog-tables-default.txt-Results]": [
{
"checksum": "aec1d30f05185d9a4d18baa033e21b73",
"size": 2453,
"uri": "https://{canondata_backend}/1942100/3f6203e58cb404b40c036990713353e16c2dde44/resource.tar.gz#test.test_pg_catalog-tables-default.txt-Results_/results.txt"
}
],
"test.test[produce-process_row_and_columns-default.txt-Debug]": [
{
"checksum": "a76d7d28c380c337f07a61a7a671ed0f",
Expand Down
2 changes: 2 additions & 0 deletions ydb/library/yql/tools/dqrun/dqrun.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,7 @@ int RunMain(int argc, const char* argv[])
IMetricsRegistryPtr metricsRegistry = CreateMetricsRegistry(GetSensorsGroupFor(NSensorComponent::kDq));
clusterMapping["plato"] = YtProviderName;
clusterMapping["pg_catalog"] = PgProviderName;
clusterMapping["information_schema"] = PgProviderName;

TString mountConfig;
TString mestricsPusherConfig;
Expand Down Expand Up @@ -696,6 +697,7 @@ int RunMain(int argc, const char* argv[])

THashMap<TString, TString> clusters;
clusters["pg_catalog"] = PgProviderName;
clusters["information_schema"] = PgProviderName;

TVector<TDataProviderInitializer> dataProvidersInit;
dataProvidersInit.push_back(GetPgDataProviderInitializer());
Expand Down
1 change: 1 addition & 0 deletions ydb/library/yql/tools/pgrun/pgrun.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1071,6 +1071,7 @@ int Main(int argc, char* argv[])
static const TString DefaultCluster{"plato"};
clusterMapping[DefaultCluster] = YtProviderName;
clusterMapping["pg_catalog"] = PgProviderName;
clusterMapping["information_schema"] = PgProviderName;

opts.AddHelpOption();
opts.AddLongOption("datadir", "directory for tables").StoreResult<TString>(&rawDataDir);
Expand Down
1 change: 1 addition & 0 deletions ydb/library/yql/tools/sql2yql/sql2yql.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ int BuildAST(int argc, char* argv[]) {
THashMap<TString, TString> clusterMapping;
clusterMapping["plato"] = NYql::YtProviderName;
clusterMapping["pg_catalog"] = NYql::PgProviderName;
clusterMapping["information_schema"] = NYql::PgProviderName;

THashMap<TString, TString> tables;
THashSet<TString> flags;
Expand Down
3 changes: 2 additions & 1 deletion ydb/library/yql/tools/yqlrun/http/yql_server.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,8 @@ NSQLTranslation::TTranslationSettings GetTranslationSettings(const THolder<TGate
static const THashMap<TString, TString> clusters = {
{ "plato", TString(YtProviderName) },
{ "plato_rtmr", TString(RtmrProviderName) },
{ "pg_catalog", TString(PgProviderName) }
{ "pg_catalog", TString(PgProviderName) },
{ "information_schema", TString(PgProviderName) },
};

NSQLTranslation::TTranslationSettings settings;
Expand Down
1 change: 1 addition & 0 deletions ydb/library/yql/tools/yqlrun/yqlrun.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,7 @@ int Main(int argc, const char *argv[])
THashSet<TString> sqlFlags;
clusterMapping["plato"] = YtProviderName;
clusterMapping["pg_catalog"] = PgProviderName;
clusterMapping["information_schema"] = PgProviderName;
ui32 progsConcurrentCount = 0;
TString paramsFile;
ui16 syntaxVersion;
Expand Down