Skip to content

Commit abd7c1a

Browse files
kardymondsblinkov
kardymonds
authored and
blinkov
committed
YQ-4084 Add available external data sources (#15192)
1 parent cf86700 commit abd7c1a

File tree

22 files changed

+320
-98
lines changed

22 files changed

+320
-98
lines changed

‎ydb/core/external_sources/external_source_factory.cpp‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,27 @@ namespace NKikimr::NExternalSource {
1414
namespace {
1515

1616
struct TExternalSourceFactory : public IExternalSourceFactory {
17-
TExternalSourceFactory(const TMap<TString, IExternalSource::TPtr>& sources)
17+
TExternalSourceFactory(
18+
const TMap<TString, IExternalSource::TPtr>& sources,
19+
const std::set<TString>& availableExternalDataSources)
1820
: Sources(sources)
21+
, AvailableExternalDataSources(availableExternalDataSources)
1922
{}
2023

2124
IExternalSource::TPtr GetOrCreate(const TString& type) const override {
2225
auto it = Sources.find(type);
23-
if (it != Sources.end()) {
24-
return it->second;
26+
if (it == Sources.end()) {
27+
throw TExternalSourceException() << "External source with type " << type << " was not found";
2528
}
26-
throw TExternalSourceException() << "External source with type " << type << " was not found";
29+
if (!AvailableExternalDataSources.contains(type)) {
30+
throw TExternalSourceException() << "External source with type " << type << " is disabled. Please contact your system administrator to enable it";
31+
}
32+
return it->second;
2733
}
2834

2935
private:
30-
TMap<TString, IExternalSource::TPtr> Sources;
36+
const TMap<TString, IExternalSource::TPtr> Sources;
37+
const std::set<TString> AvailableExternalDataSources;
3138
};
3239

3340
}
@@ -37,7 +44,8 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
3744
size_t pathsLimit,
3845
std::shared_ptr<NYql::ISecuredServiceAccountCredentialsFactory> credentialsFactory,
3946
bool enableInfer,
40-
bool allowLocalFiles) {
47+
bool allowLocalFiles,
48+
const std::set<TString>& availableExternalDataSources) {
4149
std::vector<TRegExMatch> hostnamePatternsRegEx(hostnamePatterns.begin(), hostnamePatterns.end());
4250
return MakeIntrusive<TExternalSourceFactory>(TMap<TString, IExternalSource::TPtr>{
4351
{
@@ -84,7 +92,8 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
8492
ToString(NYql::EDatabaseType::Solomon),
8593
CreateExternalDataSource(TString{NYql::SolomonProviderName}, {"NONE", "TOKEN"}, {"use_ssl", "grpc_port"}, hostnamePatternsRegEx)
8694
}
87-
});
95+
},
96+
availableExternalDataSources);
8897
}
8998

9099
}

‎ydb/core/external_sources/external_source_factory.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#include "external_source.h"
44
#include <ydb/library/yql/providers/common/token_accessor/client/factory.h>
5+
#include <ydb/library/yql/providers/common/db_id_async_resolver/database_type.h>
56

67
namespace NKikimr::NExternalSource {
78

@@ -16,6 +17,7 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
1617
size_t pathsLimit = 50000,
1718
std::shared_ptr<NYql::ISecuredServiceAccountCredentialsFactory> credentialsFactory = nullptr,
1819
bool enableInfer = false,
19-
bool allowLocalFiles = false);
20+
bool allowLocalFiles = false,
21+
const std::set<TString>& availableExternalDataSources = NYql::GetAllExternalDataSourceTypes());
2022

2123
}

‎ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
#include "kqp_federated_query_helpers.h"
22

3+
#include <ydb/library/actors/http/http_proxy.h>
4+
#include <ydb/library/yql/providers/common/db_id_async_resolver/database_type.h>
5+
36
#include <ydb/core/base/counters.h>
47
#include <ydb/core/base/feature_flags.h>
58
#include <ydb/core/protos/auth.pb.h>
@@ -56,6 +59,11 @@ namespace NKikimr::NKqp {
5659
return std::make_pair(ToString(host), scheme == "grpcs");
5760
}
5861

62+
bool IsValidExternalDataSourceType(const TString& type) {
63+
static auto allTypes = NYql::GetAllExternalDataSourceTypes();
64+
return allTypes.contains(type);
65+
}
66+
5967
// TKqpFederatedQuerySetupFactoryDefault contains network clients and service actors necessary
6068
// for federated queries. HTTP Gateway (required by S3 provider) is run by default even without
6169
// explicit configuration. Token Accessor and Connector Client are run only if config is provided.
@@ -161,6 +169,11 @@ namespace NKikimr::NKqp {
161169
return std::make_shared<TKqpFederatedQuerySetupFactoryNoop>();
162170
}
163171

172+
for (const auto& source : appConfig.GetQueryServiceConfig().GetAvailableExternalDataSources()) {
173+
if (!IsValidExternalDataSourceType(source)) {
174+
ythrow yexception() << "wrong AvailableExternalDataSources \"" << source << "\"";
175+
}
176+
}
164177
return std::make_shared<NKikimr::NKqp::TKqpFederatedQuerySetupFactoryDefault>(setup, appData, appConfig);
165178
}
166179

‎ydb/core/kqp/host/kqp_host.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1164,12 +1164,15 @@ class TKqpHost : public IKqpHost {
11641164
}
11651165

11661166
if (FederatedQuerySetup) {
1167-
ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory({},
1167+
const auto& hostnamePatterns = QueryServiceConfig.GetHostnamePatterns();
1168+
const auto& availableExternalDataSources = QueryServiceConfig.GetAvailableExternalDataSources();
1169+
ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory(std::vector<TString>(hostnamePatterns.begin(), hostnamePatterns.end()),
11681170
ActorSystem,
11691171
FederatedQuerySetup->S3GatewayConfig.GetGeneratorPathsLimit(),
11701172
FederatedQuerySetup ? FederatedQuerySetup->CredentialsFactory : nullptr,
11711173
Config->FeatureFlags.GetEnableExternalSourceSchemaInference(),
1172-
FederatedQuerySetup->S3GatewayConfig.GetAllowLocalFiles());
1174+
FederatedQuerySetup->S3GatewayConfig.GetAllowLocalFiles(),
1175+
std::set<TString>(availableExternalDataSources.cbegin(), availableExternalDataSources.cend()));
11731176
}
11741177
}
11751178

‎ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp‎

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -296,37 +296,47 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
296296
}
297297

298298
Y_UNIT_TEST(TestCreateExternalTable) {
299-
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false));
299+
NKikimrConfig::TAppConfig appCfg;
300+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
301+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false).SetAppConfig(appCfg));
300302
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
301303
TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
302304
TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table");
303305
}
304306

305307
Y_UNIT_TEST(TestCreateSameExternalTable) {
306-
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false));
308+
NKikimrConfig::TAppConfig appCfg;
309+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
310+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false).SetAppConfig(appCfg));
307311
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
308312
TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
309313
TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table");
310314
TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table", true);
311315
}
312316

313317
Y_UNIT_TEST(TestDropExternalTable) {
314-
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false));
318+
NKikimrConfig::TAppConfig appCfg;
319+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
320+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false).SetAppConfig(appCfg));
315321
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
316322
TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
317323
TestCreateExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table");
318324
TestDropExternalTable(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_table");
319325
}
320326

321327
Y_UNIT_TEST(TestDropExternalDataSource) {
322-
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false));
328+
NKikimrConfig::TAppConfig appCfg;
329+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
330+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false).SetAppConfig(appCfg));
323331
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
324332
TestCreateExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
325333
TestDropExternalDataSource(*kikimr.GetTestServer().GetRuntime(), GetIcGateway(kikimr.GetTestServer()), "/Root/f1/f2/external_data_source");
326334
}
327335

328336
Y_UNIT_TEST(TestLoadExternalTable) {
329-
TKikimrRunner kikimr;
337+
NKikimrConfig::TAppConfig appCfg;
338+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
339+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
330340
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
331341
auto db = kikimr.GetTableClient();
332342
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -366,7 +376,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
366376
}
367377

368378
Y_UNIT_TEST(TestLoadServiceAccountSecretValueFromExternalDataSourceMetadata) {
369-
TKikimrRunner kikimr;
379+
NKikimrConfig::TAppConfig appCfg;
380+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
381+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
370382
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
371383
auto db = kikimr.GetTableClient();
372384
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -404,7 +416,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
404416
}
405417

406418
Y_UNIT_TEST(TestLoadBasicSecretValueFromExternalDataSourceMetadata) {
407-
TKikimrRunner kikimr;
419+
NKikimrConfig::TAppConfig appCfg;
420+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("PostgreSQL");
421+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
408422
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
409423
auto db = kikimr.GetTableClient();
410424
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -435,7 +449,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
435449
}
436450

437451
Y_UNIT_TEST(TestLoadMdbBasicSecretValueFromExternalDataSourceMetadata) {
438-
TKikimrRunner kikimr;
452+
NKikimrConfig::TAppConfig appCfg;
453+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("PostgreSQL");
454+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
439455
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
440456
auto db = kikimr.GetTableClient();
441457
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -473,7 +489,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
473489
}
474490

475491
Y_UNIT_TEST(TestLoadAwsSecretValueFromExternalDataSourceMetadata) {
476-
TKikimrRunner kikimr;
492+
NKikimrConfig::TAppConfig appCfg;
493+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
494+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
477495
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
478496
auto db = kikimr.GetTableClient();
479497
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -510,7 +528,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
510528
}
511529

512530
Y_UNIT_TEST(TestLoadDataSourceProperties) {
513-
TKikimrRunner kikimr;
531+
NKikimrConfig::TAppConfig appCfg;
532+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("PostgreSQL");
533+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
514534
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
515535
auto db = kikimr.GetTableClient();
516536
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -558,7 +578,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
558578
}
559579

560580
Y_UNIT_TEST(TestLoadTokenSecretValueFromExternalDataSourceMetadata) {
561-
TKikimrRunner kikimr;
581+
NKikimrConfig::TAppConfig appCfg;
582+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("YT");
583+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
562584
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
563585
auto db = kikimr.GetTableClient();
564586
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -588,7 +610,9 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
588610
}
589611

590612
Y_UNIT_TEST(TestSecretsExistingValidation) {
591-
TKikimrRunner kikimr;
613+
NKikimrConfig::TAppConfig appCfg;
614+
appCfg.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
615+
TKikimrRunner kikimr(NKqp::TKikimrSettings().SetAppConfig(appCfg));
592616
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
593617
auto db = kikimr.GetTableClient();
594618
auto session = db.CreateSession().GetValueSync().GetSession();

‎ydb/core/kqp/ut/common/columnshard.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ namespace NKqp {
2121
}
2222
if (!kikimrSettings.FeatureFlags.HasEnableExternalDataSources()) {
2323
kikimrSettings.SetEnableExternalDataSources(true);
24+
kikimrSettings.AppConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
2425
}
2526

2627
Kikimr = std::make_unique<TKikimrRunner>(kikimrSettings);

‎ydb/core/kqp/ut/federated_query/common/common.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@ namespace NKikimr::NKqp::NFederatedQueryTest {
4141
if (!appConfig) {
4242
appConfig.emplace();
4343
}
44+
appConfig->MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
45+
appConfig->MutableQueryServiceConfig()->AddAvailableExternalDataSources("ClickHouse");
46+
appConfig->MutableQueryServiceConfig()->AddAvailableExternalDataSources("PostgreSQL");
47+
appConfig->MutableQueryServiceConfig()->AddAvailableExternalDataSources("MySQL");
48+
appConfig->MutableQueryServiceConfig()->AddAvailableExternalDataSources("Ydb");
4449

4550
auto settings = TKikimrSettings();
4651

‎ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,11 @@ namespace NKikimr::NKqp {
6969
appConfig.MutableQueryServiceConfig()->MutableGeneric()->MutableConnector()->MutableEndpoint()->set_host("localhost");
7070
appConfig.MutableQueryServiceConfig()->MutableGeneric()->MutableConnector()->MutableEndpoint()->set_port(1234);
7171
appConfig.MutableQueryServiceConfig()->MutableGeneric()->MutableDefaultSettings()->Add(std::move(dateTimeFormat));
72+
appConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage");
73+
appConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ClickHouse");
74+
appConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("PostgreSQL");
75+
appConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("MySQL");
76+
appConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("Ydb");
7277
return appConfig;
7378
}
7479

0 commit comments

Comments
 (0)