Skip to content

Commit 6d550e2

Browse files
kardymondskardymonds
authored
Merge 46a4685 into b0bd8f9 
2 parents b0bd8f9 + 46a4685 commit 6d550e2

File tree

8 files changed

+87
-1
lines changed

8 files changed

+87
-1
lines changed

‎ydb/core/base/appdata_fwd.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,8 @@ struct TAppData {
290290
// Tracing configurator (look for tracing config in ydb/core/jaeger_tracing/actors_tracing_control)
291291
TIntrusivePtr<NKikimr::NJaegerTracing::TSamplingThrottlingConfigurator> TracingConfigurator;
292292

293+
TSet<TString> AvailableExternalDataSources;
294+
293295
TAppData(
294296
ui32 sysPoolId, ui32 userPoolId, ui32 ioPoolId, ui32 batchPoolId,
295297
TMap<TString, ui32> servicePools,

‎ydb/core/driver_lib/run/run.cpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,13 @@ class TDomainsInitializer : public IAppDataInitializer {
256256
appData->CompactionConfig = Config.GetCompactionConfig();
257257
appData->BackgroundCleaningConfig = Config.GetBackgroundCleaningConfig();
258258
appData->DataErasureConfig = Config.GetDataErasureConfig();
259+
260+
const auto& queryServiceConfig = Config.GetQueryServiceConfig();
261+
if (queryServiceConfig.AvailableExternalDataSourcesSize() > 0) {
262+
const auto& availableSources = queryServiceConfig.GetAvailableExternalDataSources();
263+
TSet<TString> set(availableSources.cbegin(), availableSources.cend());
264+
appData->AvailableExternalDataSources = std::move(set);
265+
}
259266
}
260267
};
261268

‎ydb/core/external_sources/external_source_factory.cpp‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,4 +87,21 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
8787
});
8888
}
8989

90+
bool IsValidExternalDataSourceType(const TString& type) {
91+
static TSet<TString> allTypes = {
92+
ToString(NYql::EDatabaseType::ObjectStorage),
93+
ToString(NYql::EDatabaseType::ClickHouse),
94+
ToString(NYql::EDatabaseType::PostgreSQL),
95+
ToString(NYql::EDatabaseType::MySQL),
96+
ToString(NYql::EDatabaseType::Ydb),
97+
ToString(NYql::EDatabaseType::YT),
98+
ToString(NYql::EDatabaseType::Greenplum),
99+
ToString(NYql::EDatabaseType::MsSQLServer),
100+
ToString(NYql::EDatabaseType::Oracle),
101+
ToString(NYql::EDatabaseType::Logging),
102+
ToString(NYql::EDatabaseType::Solomon)
103+
};
104+
return allTypes.contains(type);
105+
}
106+
90107
}

‎ydb/core/external_sources/external_source_factory.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
1818
bool enableInfer = false,
1919
bool allowLocalFiles = false);
2020

21+
bool IsValidExternalDataSourceType(const TString& type);
2122
}

‎ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,18 @@ TYqlConclusion<std::pair<TString, TString>> SplitPath(const TString& tableName,
122122
return TYqlConclusionStatus::Success();
123123
}
124124

125+
[[nodiscard]] TYqlConclusionStatus CheckAvailableExternalDataSources(const NKikimrSchemeOp::TExternalDataSourceDescription& externaDataSourceDesc, const TExternalDataSourceManager::TInternalModificationContext& context) {
126+
auto* actorSystem = context.GetExternalData().GetActorSystem();
127+
if (!actorSystem) {
128+
return TYqlConclusionStatus::Fail(NYql::TIssuesIds::KIKIMR_INTERNAL_ERROR, "Internal error. EXTERNAL_DATA_SOURCE creation and drop operations needs an actor system. Please contact internal support");
129+
}
130+
auto sourceType = externaDataSourceDesc.GetSourceType();
131+
if (!AppData(actorSystem)->AvailableExternalDataSources.empty() && !AppData(actorSystem)->AvailableExternalDataSources.contains(sourceType)) {
132+
return TYqlConclusionStatus::Fail(NYql::TIssuesIds::KIKIMR_UNSUPPORTED, TStringBuilder() << "External data source " << sourceType << " are disabled. Please contact your system administrator to enable it");
133+
}
134+
return TYqlConclusionStatus::Success();
135+
}
136+
125137
[[nodiscard]] TYqlConclusionStatus ErrorFromActivityType(TExternalDataSourceManager::EActivityType activityType) {
126138
using EActivityType = TExternalDataSourceManager::EActivityType;
127139

@@ -209,7 +221,13 @@ TYqlConclusionStatus TExternalDataSourceManager::PrepareCreateExternalDataSource
209221
schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateExternalDataSource);
210222
schemeTx.SetFailedOnAlreadyExists(!settings.GetExistingOk());
211223

212-
return FillCreateExternalDataSourceDesc(*schemeTx.MutableCreateExternalDataSource(), name, settings);
224+
if (auto status = FillCreateExternalDataSourceDesc(*schemeTx.MutableCreateExternalDataSource(), name, settings); status.IsFail()) {
225+
return status;
226+
}
227+
if (auto status = CheckAvailableExternalDataSources(schemeTx.GetCreateExternalDataSource(), context); status.IsFail()) {
228+
return status;
229+
}
230+
return TYqlConclusionStatus::Success();
213231
}
214232

215233
TYqlConclusionStatus TExternalDataSourceManager::PrepareDropExternalDataSource(NKqpProto::TKqpSchemeOperation& schemeOperation, const NYql::TDropObjectSettings& settings, TInternalModificationContext& context) const {

‎ydb/core/kqp/host/kqp_host.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1163,6 +1163,12 @@ class TKqpHost : public IKqpHost {
11631163
SessionCtx->SetTempTables(std::move(tempTablesState));
11641164
}
11651165

1166+
for (const auto& source: queryServiceConfig.GetAvailableExternalDataSources()) {
1167+
if (!NKikimr::NExternalSource::IsValidExternalDataSourceType(source)) {
1168+
ythrow yexception() << "wrong AvailableExternalDataSources \"" << source << "\"";
1169+
}
1170+
}
1171+
11661172
if (FederatedQuerySetup) {
11671173
ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory({},
11681174
ActorSystem,

‎ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6617,6 +6617,40 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
66176617
UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), "External data sources are disabled. Please contact your system administrator to enable it", result.GetIssues().ToString());
66186618
}
66196619

6620+
Y_UNIT_TEST(DisableS3ExternalDataSource) {
6621+
TKikimrRunner kikimr;
6622+
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
6623+
kikimr.GetTestServer().GetRuntime()->GetAppData(0).AvailableExternalDataSources.insert("PostgreSQL");
6624+
6625+
auto db = kikimr.GetTableClient();
6626+
auto session = db.CreateSession().GetValueSync().GetSession();
6627+
TString externalDataSourceName = "/Root/ExternalDataSource";
6628+
auto query = TStringBuilder() << R"(
6629+
CREATE EXTERNAL DATA SOURCE `)" << externalDataSourceName << R"(` WITH (
6630+
SOURCE_TYPE="ObjectStorage",
6631+
LOCATION="my-bucket",
6632+
AUTH_METHOD="NONE"
6633+
);)";
6634+
auto result = session.ExecuteSchemeQuery(query).GetValueSync();
6635+
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
6636+
UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), "External data source ObjectStorage are disabled. Please contact your system administrator to enable it", result.GetIssues().ToString());
6637+
6638+
auto query2 = TStringBuilder() << R"(
6639+
CREATE OBJECT `baz` (TYPE SECRET) WITH value=`MySecretData`;
6640+
6641+
CREATE EXTERNAL DATA SOURCE `)" << externalDataSourceName << R"(` WITH (
6642+
SOURCE_TYPE="PostgreSQL",
6643+
LOCATION="my-bucket",
6644+
AUTH_METHOD="BASIC",
6645+
LOGIN="admin",
6646+
PASSWORD_SECRET_NAME = "baz",
6647+
DATABASE_NAME="cheburashka"
6648+
);)";
6649+
result = session.ExecuteSchemeQuery(query2).GetValueSync();
6650+
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
6651+
6652+
}
6653+
66206654
Y_UNIT_TEST(CreateExternalDataSourceValidationAuthMethod) {
66216655
TKikimrRunner kikimr;
66226656
kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);

‎ydb/core/protos/config.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1077,6 +1077,7 @@ message TQueryServiceConfig {
10771077
optional uint64 ProgressStatsPeriodMs = 14 [default = 0]; // 0 = disabled
10781078
optional uint32 QueryTimeoutDefaultSeconds = 19 [default = 1800];
10791079
optional bool EnableMatchRecognize = 20 [default = false];
1080+
repeated string AvailableExternalDataSources = 22; // List of external data source types; enabled all if empty
10801081
}
10811082

10821083
// Config describes immediate controls and allows

0 commit comments

Comments
 (0)