Skip to content

Commit 2021b43

Browse files
doroolegdorooleg
authored
Merge c5f65c2 into 6cf4566 
2 parents 6cf4566 + c5f65c2 commit 2021b43

File tree

9 files changed

+38
-57
lines changed

9 files changed

+38
-57
lines changed

‎ydb/library/yql/providers/generic/connector/libcpp/error.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ namespace NYql::NConnector {
3737
return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_UNSUPPORTED;
3838
case ::Ydb::StatusIds::StatusCode::StatusIds_StatusCode_NOT_FOUND:
3939
return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_BAD_REQUEST;
40+
case ::Ydb::StatusIds::StatusCode::StatusIds_StatusCode_SCHEME_ERROR:
41+
return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_SCHEME_ERROR;
4042
default:
4143
ythrow yexception() << "Unexpected YDB status code: " << ::Ydb::StatusIds::StatusCode_Name(error.status());
4244
}

‎ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ services:
1212
- 8123
1313
fq-connector-go:
1414
container_name: fq-tests-ch-fq-connector-go
15-
image: ghcr.io/ydb-platform/fq-connector-go:v0.2.20@sha256:a1771f348dc8be6219865e332f788429907cdfec3677b3e98f0bc6f7dd542dc6
15+
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
1616
ports:
1717
- 2130
1818
volumes:

‎ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
services:
22
fq-connector-go:
33
container_name: fq-tests-pg-fq-connector-go
4-
image: ghcr.io/ydb-platform/fq-connector-go:v0.2.20@sha256:a1771f348dc8be6219865e332f788429907cdfec3677b3e98f0bc6f7dd542dc6
4+
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
55
ports:
66
- 2130
77
volumes:

‎ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ services:
55
echo \"$$(dig fq-tests-ydb-ydb +short) fq-tests-ydb-ydb\" >> /etc/hosts; cat /etc/hosts;
66
/opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml"
77
container_name: fq-tests-ydb-fq-connector-go
8-
image: ghcr.io/ydb-platform/fq-connector-go:v0.2.20@sha256:a1771f348dc8be6219865e332f788429907cdfec3677b3e98f0bc6f7dd542dc6
8+
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
99
ports:
1010
- 2130
1111
volumes:

‎ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ services:
1212
- 8123
1313
fq-connector-go:
1414
container_name: fq-tests-join-fq-connector-go
15-
image: ghcr.io/ydb-platform/fq-connector-go:v0.2.20@sha256:a1771f348dc8be6219865e332f788429907cdfec3677b3e98f0bc6f7dd542dc6
15+
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
1616
ports:
1717
- 2130
1818
volumes:

‎ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -107,14 +107,12 @@ namespace NYql {
107107
NYql::TGenericClusterConfig& clusterConfig) {
108108
auto it = properties.find("database_name");
109109
if (it == properties.cend()) {
110-
// TODO: make this property required during https://st.yandex-team.ru/YQ-2494
111-
// ythrow yexception() << "missing 'DATABASE_NAME' value";
110+
// DATABASE_NAME is a mandatory field for the most of databases,
111+
// however, managed YDB does not require it, so we have to accept empty values here.
112112
return;
113113
}
114114

115115
if (!it->second) {
116-
// TODO: make this property required during https://st.yandex-team.ru/YQ-2494
117-
// ythrow yexception() << "invalid 'DATABASE_NAME' value: '" << it->second << "'";
118116
return;
119117
}
120118

@@ -125,14 +123,12 @@ namespace NYql {
125123
NYql::TGenericClusterConfig& clusterConfig) {
126124
auto it = properties.find("schema");
127125
if (it == properties.cend()) {
128-
// TODO: make this property required during https://st.yandex-team.ru/YQ-2494
129-
// ythrow yexception() << "missing 'SCHEMA' value";
126+
// SCHEMA is optional field
130127
return;
131128
}
132129

133130
if (!it->second) {
134-
// TODO: make this property required during https://st.yandex-team.ru/YQ-2494
135-
// ythrow yexception() << "invalid 'SCHEMA' value: '" << it->second << "'";
131+
// SCHEMA is optional field
136132
return;
137133
}
138134

@@ -318,9 +314,20 @@ namespace NYql {
318314
}
319315

320316
static const TSet<NConnector::NApi::EDataSourceKind> managedDatabaseKinds{
317+
NConnector::NApi::EDataSourceKind::CLICKHOUSE,
318+
NConnector::NApi::EDataSourceKind::GREENPLUM,
319+
NConnector::NApi::EDataSourceKind::MYSQL,
321320
NConnector::NApi::EDataSourceKind::POSTGRESQL,
321+
NConnector::NApi::EDataSourceKind::YDB,
322+
};
323+
324+
static const TSet<NConnector::NApi::EDataSourceKind> traditionalRelationalDatabaseKinds{
322325
NConnector::NApi::EDataSourceKind::CLICKHOUSE,
323-
NConnector::NApi::EDataSourceKind::YDB};
326+
NConnector::NApi::EDataSourceKind::GREENPLUM,
327+
NConnector::NApi::EDataSourceKind::MS_SQL_SERVER,
328+
NConnector::NApi::EDataSourceKind::MYSQL,
329+
NConnector::NApi::EDataSourceKind::POSTGRESQL,
330+
};
324331

325332
void ValidateGenericClusterConfig(
326333
const NYql::TGenericClusterConfig& clusterConfig,
@@ -396,6 +403,17 @@ namespace NYql {
396403
}
397404
}
398405

406+
// All the databases with exception to managed YDB:
407+
// * DATABASE_NAME is mandatory field
408+
if (traditionalRelationalDatabaseKinds.contains(clusterConfig.GetKind())) {
409+
if (!clusterConfig.GetDatabaseName()) {
410+
return ValidationError(
411+
clusterConfig,
412+
context,
413+
"You must provide database name explicitly");
414+
}
415+
}
416+
399417
// check required fields
400418
if (!clusterConfig.GetName()) {
401419
return ValidationError(clusterConfig, context, "empty field 'Name'");

‎ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -102,15 +102,6 @@ namespace NYql {
102102
const auto& clusterConfig = State_->Configuration->ClusterNamesToClusterConfigs[clusterName];
103103
const auto& endpoint = clusterConfig.endpoint();
104104

105-
Generic::TSource source;
106-
107-
// for backward compability full path can be used (cluster_name.`db_name.table`)
108-
// TODO: simplify during https://st.yandex-team.ru/YQ-2494
109-
TStringBuf db, dbTable;
110-
if (!TStringBuf(table).TrySplit('.', db, dbTable)) {
111-
dbTable = table;
112-
}
113-
114105
YQL_CLOG(INFO, ProviderGeneric)
115106
<< "Filling source settings"
116107
<< ": cluster: " << clusterName
@@ -125,8 +116,9 @@ namespace NYql {
125116
}
126117

127118
// prepare select
119+
Generic::TSource source;
128120
auto select = source.mutable_select();
129-
select->mutable_from()->set_table(TString(dbTable));
121+
select->mutable_from()->set_table(table);
130122
select->mutable_data_source_instance()->CopyFrom(tableMeta.value()->DataSourceInstance);
131123

132124
auto items = select->mutable_what()->mutable_items();
@@ -264,7 +256,6 @@ namespace NYql {
264256
void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) override {
265257
RegisterDqGenericMkqlCompilers(compiler, State_);
266258
}
267-
268259
private:
269260
const TGenericState::TPtr State_;
270261
};

‎ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp‎

Lines changed: 2 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -367,38 +367,8 @@ namespace NYql {
367367

368368
void FillTablePath(NConnector::NApi::TDescribeTableRequest& request, const TGenericClusterConfig& clusterConfig,
369369
const TString& tablePath) {
370-
// for backward compability full path can be used (cluster_name.`db_name.table`)
371-
// TODO: simplify during https://st.yandex-team.ru/YQ-2494
372-
const auto dataSourceKind = clusterConfig.GetKind();
373-
const auto& dbNameFromConfig = clusterConfig.GetDatabaseName();
374-
TStringBuf dbNameTarget, tableName;
375-
auto isFullPath = TStringBuf(tablePath).TrySplit('.', dbNameTarget, tableName);
376-
377-
if (!dbNameFromConfig.empty()) {
378-
dbNameTarget = dbNameFromConfig;
379-
if (!isFullPath) {
380-
tableName = tablePath;
381-
}
382-
} else if (!isFullPath) {
383-
tableName = tablePath;
384-
switch (dataSourceKind) {
385-
case NYql::NConnector::NApi::CLICKHOUSE:
386-
dbNameTarget = "default";
387-
break;
388-
case NYql::NConnector::NApi::POSTGRESQL:
389-
dbNameTarget = "postgres";
390-
break;
391-
case NYql::NConnector::NApi::MS_SQL_SERVER:
392-
dbNameTarget = "mssqlserver";
393-
break;
394-
default:
395-
ythrow yexception() << "You must provide database name explicitly for data source kind: '"
396-
<< NYql::NConnector::NApi::EDataSourceKind_Name(dataSourceKind) << "'";
397-
}
398-
} // else take database name from table path
399-
400-
request.mutable_data_source_instance()->set_database(TString(dbNameTarget));
401-
request.set_table(TString(tableName));
370+
request.mutable_data_source_instance()->set_database(clusterConfig.GetDatabaseName());
371+
request.set_table(tablePath);
402372
}
403373

404374
private:

‎ydb/tests/fq/generic/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ services:
1515
echo \"$$(dig tests-fq-generic-ydb +short) tests-fq-generic-ydb\" >> /etc/hosts; cat /etc/hosts;
1616
/opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml"
1717
container_name: tests-fq-generic-fq-connector-go
18-
image: ghcr.io/ydb-platform/fq-connector-go:v0.2.20@sha256:a1771f348dc8be6219865e332f788429907cdfec3677b3e98f0bc6f7dd542dc6
18+
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
1919
ports:
2020
- "2130"
2121
postgresql:

0 commit comments

Comments
 (0)