Skip to content

Commit ce2c127

Browse files
lifthelmlifthelm
authored
Merge 1a9ab3c into f7e7be8 
2 parents f7e7be8 + 1a9ab3c commit ce2c127

File tree

11 files changed

+41
-57
lines changed

11 files changed

+41
-57
lines changed

‎ydb/library/yql/providers/generic/connector/libcpp/error.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ namespace NYql::NConnector {
3737
return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_UNSUPPORTED;
3838
case ::Ydb::StatusIds::StatusCode::StatusIds_StatusCode_NOT_FOUND:
3939
return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_BAD_REQUEST;
40+
case ::Ydb::StatusIds::StatusCode::StatusIds_StatusCode_SCHEME_ERROR:
41+
return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_SCHEME_ERROR;
4042
default:
4143
ythrow yexception() << "Unexpected YDB status code: " << ::Ydb::StatusIds::StatusCode_Name(error.status());
4244
}

‎ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ services:
1212
- 8123
1313
fq-connector-go:
1414
container_name: fq-tests-ch-fq-connector-go
15-
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.17@sha256:3763344ab70f9a6b8c1546f15c0b31465590e8ac6636be15ca2d29c4f4cd9b19
15+
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
1616
ports:
1717
- 2130
1818
volumes:

‎ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
services:
22
fq-connector-go:
33
container_name: fq-tests-my-fq-connector-go
4-
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.17@sha256:3763344ab70f9a6b8c1546f15c0b31465590e8ac6636be15ca2d29c4f4cd9b19
4+
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
55
ports:
66
- 2130
77
volumes:

‎ydb/library/yql/providers/generic/connector/tests/datasource/mysql/select_positive.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,12 +140,12 @@ def _primitive_types(self) -> Sequence[TestCase]:
140140
# Don't be surprised - binary types look like UTF8 strings in Go
141141
Column(
142142
name='col_22_binary',
143-
ydb_type=makeOptionalYdbTypeFromTypeID(Type.UTF8),
143+
ydb_type=makeOptionalYdbTypeFromTypeID(Type.STRING),
144144
data_source_type=DataSourceType(my=mysql.Binary()),
145145
),
146146
Column(
147147
name='col_23_varbinary',
148-
ydb_type=makeOptionalYdbTypeFromTypeID(Type.UTF8),
148+
ydb_type=makeOptionalYdbTypeFromTypeID(Type.STRING),
149149
data_source_type=DataSourceType(my=mysql.VarBinary()),
150150
),
151151
Column(

‎ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
services:
22
fq-connector-go:
33
container_name: fq-tests-pg-fq-connector-go
4-
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.17@sha256:3763344ab70f9a6b8c1546f15c0b31465590e8ac6636be15ca2d29c4f4cd9b19
4+
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
55
ports:
66
- 2130
77
volumes:

‎ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ services:
55
echo \"$$(dig fq-tests-ydb-ydb +short) fq-tests-ydb-ydb\" >> /etc/hosts; cat /etc/hosts;
66
/opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml"
77
container_name: fq-tests-ydb-fq-connector-go
8-
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.17@sha256:3763344ab70f9a6b8c1546f15c0b31465590e8ac6636be15ca2d29c4f4cd9b19
8+
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
99
ports:
1010
- 2130
1111
volumes:

‎ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ services:
1212
- 8123
1313
fq-connector-go:
1414
container_name: fq-tests-join-fq-connector-go
15-
image: ghcr.io/ydb-platform/fq-connector-go:v0.4.17@sha256:3763344ab70f9a6b8c1546f15c0b31465590e8ac6636be15ca2d29c4f4cd9b19
15+
image: ghcr.io/ydb-platform/fq-connector-go:v0.5.0@sha256:6d3cec43478bef88dda195cd38c10e4df719c8ce6d13c9bd288c7ec40410e9d8
1616
ports:
1717
- 2130
1818
volumes:

‎ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -107,14 +107,12 @@ namespace NYql {
107107
NYql::TGenericClusterConfig& clusterConfig) {
108108
auto it = properties.find("database_name");
109109
if (it == properties.cend()) {
110-
// TODO: make this property required during https://st.yandex-team.ru/YQ-2494
111-
// ythrow yexception() << "missing 'DATABASE_NAME' value";
110+
// DATABASE_NAME is a mandatory field for the most of databases,
111+
// however, managed YDB does not require it, so we have to accept empty values here.
112112
return;
113113
}
114114

115115
if (!it->second) {
116-
// TODO: make this property required during https://st.yandex-team.ru/YQ-2494
117-
// ythrow yexception() << "invalid 'DATABASE_NAME' value: '" << it->second << "'";
118116
return;
119117
}
120118

@@ -125,14 +123,12 @@ namespace NYql {
125123
NYql::TGenericClusterConfig& clusterConfig) {
126124
auto it = properties.find("schema");
127125
if (it == properties.cend()) {
128-
// TODO: make this property required during https://st.yandex-team.ru/YQ-2494
129-
// ythrow yexception() << "missing 'SCHEMA' value";
126+
// SCHEMA is optional field
130127
return;
131128
}
132129

133130
if (!it->second) {
134-
// TODO: make this property required during https://st.yandex-team.ru/YQ-2494
135-
// ythrow yexception() << "invalid 'SCHEMA' value: '" << it->second << "'";
131+
// SCHEMA is optional field
136132
return;
137133
}
138134

@@ -333,9 +329,21 @@ namespace NYql {
333329
}
334330

335331
static const TSet<NConnector::NApi::EDataSourceKind> managedDatabaseKinds{
332+
NConnector::NApi::EDataSourceKind::CLICKHOUSE,
333+
NConnector::NApi::EDataSourceKind::GREENPLUM,
334+
NConnector::NApi::EDataSourceKind::MYSQL,
336335
NConnector::NApi::EDataSourceKind::POSTGRESQL,
336+
NConnector::NApi::EDataSourceKind::YDB,
337+
};
338+
339+
static const TSet<NConnector::NApi::EDataSourceKind> traditionalRelationalDatabaseKinds{
337340
NConnector::NApi::EDataSourceKind::CLICKHOUSE,
338-
NConnector::NApi::EDataSourceKind::YDB};
341+
NConnector::NApi::EDataSourceKind::GREENPLUM,
342+
NConnector::NApi::EDataSourceKind::MS_SQL_SERVER,
343+
NConnector::NApi::EDataSourceKind::MYSQL,
344+
NConnector::NApi::EDataSourceKind::ORACLE,
345+
NConnector::NApi::EDataSourceKind::POSTGRESQL,
346+
};
339347

340348
void ValidateGenericClusterConfig(
341349
const NYql::TGenericClusterConfig& clusterConfig,
@@ -422,6 +430,17 @@ namespace NYql {
422430
}
423431
}
424432

433+
// All the databases with exception to managed YDB:
434+
// * DATABASE_NAME is mandatory field
435+
if (traditionalRelationalDatabaseKinds.contains(clusterConfig.GetKind())) {
436+
if (!clusterConfig.GetDatabaseName()) {
437+
return ValidationError(
438+
clusterConfig,
439+
context,
440+
"You must provide database name explicitly");
441+
}
442+
}
443+
425444
// check required fields
426445
if (!clusterConfig.GetName()) {
427446
return ValidationError(clusterConfig, context, "empty field 'Name'");

‎ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -269,13 +269,6 @@ namespace NYql {
269269
const auto& clusterConfig = State_->Configuration->ClusterNamesToClusterConfigs[clusterName];
270270
const auto& endpoint = clusterConfig.endpoint();
271271

272-
// for backward compability full path can be used (cluster_name.`db_name.table`)
273-
// TODO: simplify during https://st.yandex-team.ru/YQ-2494
274-
TStringBuf db, dbTable;
275-
if (!TStringBuf(table).TrySplit('.', db, dbTable)) {
276-
dbTable = table;
277-
}
278-
279272
YQL_CLOG(INFO, ProviderGeneric)
280273
<< "Filling lookup source settings"
281274
<< ": cluster: " << clusterName
@@ -288,7 +281,7 @@ namespace NYql {
288281
}
289282

290283
Generic::TLookupSource source;
291-
source.set_table(TString(dbTable));
284+
source.set_table(table);
292285
*source.mutable_data_source_instance() = tableMeta.value()->DataSourceInstance;
293286

294287
// Managed YDB supports access via IAM token.

‎ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp‎

Lines changed: 2 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -378,38 +378,8 @@ namespace NYql {
378378

379379
void FillTablePath(NConnector::NApi::TDescribeTableRequest& request, const TGenericClusterConfig& clusterConfig,
380380
const TString& tablePath) {
381-
// for backward compability full path can be used (cluster_name.`db_name.table`)
382-
// TODO: simplify during https://st.yandex-team.ru/YQ-2494
383-
const auto dataSourceKind = clusterConfig.GetKind();
384-
const auto& dbNameFromConfig = clusterConfig.GetDatabaseName();
385-
TStringBuf dbNameTarget, tableName;
386-
auto isFullPath = TStringBuf(tablePath).TrySplit('.', dbNameTarget, tableName);
387-
388-
if (!dbNameFromConfig.empty()) {
389-
dbNameTarget = dbNameFromConfig;
390-
if (!isFullPath) {
391-
tableName = tablePath;
392-
}
393-
} else if (!isFullPath) {
394-
tableName = tablePath;
395-
switch (dataSourceKind) {
396-
case NYql::NConnector::NApi::CLICKHOUSE:
397-
dbNameTarget = "default";
398-
break;
399-
case NYql::NConnector::NApi::POSTGRESQL:
400-
dbNameTarget = "postgres";
401-
break;
402-
case NYql::NConnector::NApi::MS_SQL_SERVER:
403-
dbNameTarget = "mssqlserver";
404-
break;
405-
default:
406-
ythrow yexception() << "You must provide database name explicitly for data source kind: '"
407-
<< NYql::NConnector::NApi::EDataSourceKind_Name(dataSourceKind) << "'";
408-
}
409-
} // else take database name from table path
410-
411-
request.mutable_data_source_instance()->set_database(TString(dbNameTarget));
412-
request.set_table(TString(tableName));
381+
request.mutable_data_source_instance()->set_database(clusterConfig.GetDatabaseName());
382+
request.set_table(tablePath);
413383
}
414384

415385
private:

0 commit comments

Comments
 (0)