Skip to content

Commit a580b00

Browse files
doroolegdorooleg
authored
Supported the ability to grant access to external data sources (#10848)
1 parent d30aab1 commit a580b00

File tree

7 files changed

+140
-70
lines changed

7 files changed

+140
-70
lines changed

‎ydb/core/fq/libs/compute/common/config.h‎

Lines changed: 34 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,25 @@ class TComputeConfig {
6868
}
6969
}
7070

71+
TVector<TString> GetExternalSourcesAccessSIDs(const TString& scope) const {
72+
const auto& controlPlane = ComputeConfig.GetYdb().GetControlPlane();
73+
switch (controlPlane.type_case()) {
74+
case NConfig::TYdbComputeControlPlane::TYPE_NOT_SET:
75+
return {};
76+
case NConfig::TYdbComputeControlPlane::kSingle:
77+
return {};
78+
case NConfig::TYdbComputeControlPlane::kCms:
79+
return GetExternalSourcesAccessSIDs(scope, controlPlane.GetCms().GetDatabaseMapping());
80+
case NConfig::TYdbComputeControlPlane::kYdbcp:
81+
return GetExternalSourcesAccessSIDs(scope, controlPlane.GetYdbcp().GetDatabaseMapping());
82+
}
83+
}
84+
85+
TVector<TString> GetExternalSourcesAccessSIDs(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
86+
const auto protoExternalSourcesAccessSIDs = GetComputeDatabaseConfig(scope, databaseMapping).GetAccessConfig().GetExternalSourcesAccessSID();
87+
return TVector<TString>{protoExternalSourcesAccessSIDs.begin(), protoExternalSourcesAccessSIDs.end()};
88+
}
89+
7190
NFq::NConfig::TYdbStorageConfig GetControlPlaneConnection(const TString& scope) const {
7291
const auto& controlPlane = ComputeConfig.GetYdb().GetControlPlane();
7392
switch (controlPlane.type_case()) {
@@ -83,15 +102,8 @@ class TComputeConfig {
83102
}
84103

85104
NFq::NConfig::TYdbStorageConfig GetControlPlaneConnection(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
86-
auto it = databaseMapping.GetScopeToComputeDatabase().find(scope);
87-
if (it != databaseMapping.GetScopeToComputeDatabase().end()) {
88-
return it->second.GetControlPlaneConnection();
89-
}
90-
return databaseMapping.GetCommon().empty()
91-
? NFq::NConfig::TYdbStorageConfig{}
92-
: databaseMapping
93-
.GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size())
94-
.GetControlPlaneConnection();
105+
auto computeDatabaseConfig = GetComputeDatabaseConfig(scope, databaseMapping);
106+
return computeDatabaseConfig.GetControlPlaneConnection();
95107
}
96108

97109
NFq::NConfig::TYdbStorageConfig GetExecutionConnection(const TString& scope) const {
@@ -137,18 +149,11 @@ class TComputeConfig {
137149
}
138150

139151
NFq::NConfig::TYdbStorageConfig GetExecutionConnection(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
140-
auto it = databaseMapping.GetScopeToComputeDatabase().find(scope);
141-
if (it != databaseMapping.GetScopeToComputeDatabase().end()) {
142-
return it->second.GetExecutionConnection();
143-
}
144-
return databaseMapping.GetCommon().empty()
145-
? NFq::NConfig::TYdbStorageConfig{}
146-
: databaseMapping
147-
.GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size())
148-
.GetExecutionConnection();
152+
auto computeDatabaseConfig = GetComputeDatabaseConfig(scope, databaseMapping);
153+
return computeDatabaseConfig.GetExecutionConnection();
149154
}
150155

151-
NFq::NConfig::TWorkloadManagerConfig GetWorkloadManagerConfig(const TString& scope) {
156+
NFq::NConfig::TWorkloadManagerConfig GetWorkloadManagerConfig(const TString& scope) const {
152157
const auto& controlPlane = ComputeConfig.GetYdb().GetControlPlane();
153158
switch (controlPlane.type_case()) {
154159
case NConfig::TYdbComputeControlPlane::TYPE_NOT_SET:
@@ -163,15 +168,19 @@ class TComputeConfig {
163168
}
164169

165170
NFq::NConfig::TWorkloadManagerConfig GetWorkloadManagerConfig(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
171+
auto computeDatabaseConfig = GetComputeDatabaseConfig(scope, databaseMapping);
172+
return computeDatabaseConfig.GetWorkloadManagerConfig();
173+
}
174+
175+
NFq::NConfig::TComputeDatabaseConfig GetComputeDatabaseConfig(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
166176
auto it = databaseMapping.GetScopeToComputeDatabase().find(scope);
167177
if (it != databaseMapping.GetScopeToComputeDatabase().end()) {
168-
return it->second.GetWorkloadManagerConfig();
178+
return it->second;
179+
}
180+
if (databaseMapping.GetCommon().empty()) {
181+
return NFq::NConfig::TComputeDatabaseConfig{};
169182
}
170-
return databaseMapping.GetCommon().empty()
171-
? NFq::NConfig::TWorkloadManagerConfig{}
172-
: databaseMapping
173-
.GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size())
174-
.GetWorkloadManagerConfig();
183+
return databaseMapping.GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size());
175184
}
176185

177186
bool YdbComputeControlPlaneEnabled(const TString& scope) const {

‎ydb/core/fq/libs/config/protos/compute.proto‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ message TWorkloadManagerConfig {
4242
repeated TResourcePool ResourcePool = 3;
4343
}
4444

45+
message TAccessConfig {
46+
repeated string ExternalSourcesAccessSID = 1;
47+
}
48+
4549
message TComputeDatabaseConfig {
4650
string Id = 7;
4751
TYdbStorageConfig ControlPlaneConnection = 1;
@@ -50,6 +54,7 @@ message TComputeDatabaseConfig {
5054
string Tenant = 2;
5155
TLoadControlConfig LoadControlConfig = 4;
5256
TWorkloadManagerConfig WorkloadManagerConfig = 5;
57+
TAccessConfig AccessConfig = 8;
5358
}
5459

5560
message TDatabaseMapping {

‎ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp‎

Lines changed: 79 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,61 @@ TString MakeSecretKeyName(const TString& prefix, const TString& folderId, const
1717
return TStringBuilder{} << prefix << "_" << folderId << "_" << name;
1818
}
1919

20+
TString MakeCreateSecretObjectSql(const TString& name, const TString& value) {
21+
using namespace fmt::literals;
22+
return fmt::format(
23+
R"(
24+
UPSERT OBJECT {name} (TYPE SECRET) WITH value={value};
25+
)",
26+
"name"_a = EncloseAndEscapeString(name, '`'),
27+
"value"_a = EncloseSecret(EncloseAndEscapeString(value, '"')));
28+
}
29+
30+
TString MakeCreateSecretAccessObjectSql(const TString& name, const TString& sid) {
31+
using namespace fmt::literals;
32+
TString accessName = TStringBuilder{} << name << ":" << sid;
33+
return fmt::format(
34+
R"(
35+
UPSERT OBJECT {name} (TYPE SECRET_ACCESS);
36+
)",
37+
"name"_a = EncloseAndEscapeString(accessName, '`'));
38+
}
39+
40+
TString MakeCreateSecretAccessObjectsSql(const TString& name, const TVector<TString>& externalSourcesAccessSIDs) {
41+
TStringBuilder result;
42+
for (const auto& sid : externalSourcesAccessSIDs) {
43+
result << MakeCreateSecretAccessObjectSql(name, sid);
44+
}
45+
return result;
46+
}
47+
48+
TString MakeDropSecretObjectSql(const TString& name) {
49+
using namespace fmt::literals;
50+
return fmt::format(
51+
R"(
52+
DROP OBJECT {name} (TYPE SECRET);
53+
)",
54+
"name"_a = EncloseAndEscapeString(name, '`'));
55+
}
56+
57+
TString MakeDropAccessSecretObjectSql(const TString& name, const TString& sid) {
58+
using namespace fmt::literals;
59+
TString accessName = TStringBuilder{} << name << ":" << sid;
60+
return fmt::format(
61+
R"(
62+
DROP OBJECT {name} (TYPE SECRET_ACCESS);
63+
)",
64+
"name"_a = EncloseAndEscapeString(accessName, '`'));
65+
}
66+
67+
TString MakeDropSecretAccessObjectsSql(const TString& name, const TVector<TString>& externalSourcesAccessSIDs) {
68+
TStringBuilder result;
69+
for (const auto& sid : externalSourcesAccessSIDs) {
70+
result << MakeDropAccessSecretObjectSql(name, sid);
71+
}
72+
return result;
73+
}
74+
2075
}
2176

2277
TString MakeCreateExternalDataTableQuery(const FederatedQuery::BindingContent& content,
@@ -100,33 +155,30 @@ TString SignAccountId(const TString& id, const TSigner::TPtr& signer) {
100155
return signer ? signer->SignAccountId(id) : TString{};
101156
}
102157

158+
159+
103160
TMaybe<TString> CreateSecretObjectQuery(const FederatedQuery::ConnectionSetting& setting,
104-
const TString& name,
105-
const TSigner::TPtr& signer,
106-
const TString& folderId) {
107-
using namespace fmt::literals;
108-
TString secretObjects;
161+
const TString& name,
162+
const TSigner::TPtr& signer,
163+
const TString& folderId,
164+
const TVector<TString>& externalSourcesAccessSIDs) {
165+
TStringBuilder result;
109166
auto serviceAccountId = ExtractServiceAccountId(setting);
110-
if (serviceAccountId) {
111-
secretObjects = signer ? fmt::format(
112-
R"(
113-
UPSERT OBJECT {sa_secret_name} (TYPE SECRET) WITH value={signature};
114-
)",
115-
"sa_secret_name"_a = EncloseAndEscapeString(MakeSecretKeyName("f1", folderId, name), '`'),
116-
"signature"_a = EncloseSecret(EncloseAndEscapeString(SignAccountId(serviceAccountId, signer), '"'))) : std::string{};
167+
if (serviceAccountId && signer) {
168+
const TString saSecretName = MakeSecretKeyName("f1", folderId, name);
169+
const TString signature = SignAccountId(serviceAccountId, signer);
170+
result << MakeCreateSecretObjectSql(saSecretName, signature);
171+
result << MakeCreateSecretAccessObjectsSql(saSecretName, externalSourcesAccessSIDs);
117172
}
118173

119174
auto password = GetPassword(setting);
120175
if (password) {
121-
secretObjects += fmt::format(
122-
R"(
123-
UPSERT OBJECT {password_secret_name} (TYPE SECRET) WITH value={password};
124-
)",
125-
"password_secret_name"_a = EncloseAndEscapeString(MakeSecretKeyName("f2", folderId, name), '`'),
126-
"password"_a = EncloseSecret(EncloseAndEscapeString(*password, '"')));
176+
const TString passwordSecretName = MakeSecretKeyName("f2", folderId, name);
177+
result << MakeCreateSecretObjectSql(passwordSecretName, *password);
178+
result << MakeCreateSecretAccessObjectsSql(passwordSecretName, externalSourcesAccessSIDs);
127179
}
128180

129-
return secretObjects ? secretObjects : TMaybe<TString>{};
181+
return result ? result : TMaybe<TString>{};
130182
}
131183

132184
TString CreateAuthParamsQuery(const FederatedQuery::ConnectionSetting& setting,
@@ -301,21 +353,14 @@ TString MakeCreateExternalDataSourceQuery(
301353
folderId));
302354
}
303355

304-
TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId) {
305-
using namespace fmt::literals;
306-
return fmt::format(
307-
R"(
308-
DROP OBJECT {secret_name1} (TYPE SECRET);
309-
DROP OBJECT {secret_name2} (TYPE SECRET);
310-
DROP OBJECT {secret_name3} (TYPE SECRET); -- for backward compatibility
311-
DROP OBJECT {secret_name4} (TYPE SECRET); -- for backward compatibility
312-
DROP OBJECT {secret_name5} (TYPE SECRET); -- for backward compatibility
313-
)",
314-
"secret_name1"_a = EncloseAndEscapeString(MakeSecretKeyName("f1", folderId, name), '`'),
315-
"secret_name2"_a = EncloseAndEscapeString(MakeSecretKeyName("f2", folderId, name), '`'),
316-
"secret_name3"_a = EncloseAndEscapeString(TStringBuilder{} << "k1" << name, '`'),
317-
"secret_name4"_a = EncloseAndEscapeString(TStringBuilder{} << "k2" << name, '`'),
318-
"secret_name5"_a = EncloseAndEscapeString(name, '`'));
356+
TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId, const TVector<TString>& externalSourcesAccessSIDs) {
357+
const TString secretName1 = MakeSecretKeyName("f1", folderId, name);
358+
const TString secretName2 = MakeSecretKeyName("f2", folderId, name);
359+
return TStringBuilder{}
360+
<< MakeDropSecretAccessObjectsSql(secretName1, externalSourcesAccessSIDs)
361+
<< MakeDropSecretObjectSql(secretName1)
362+
<< MakeDropSecretAccessObjectsSql(secretName2, externalSourcesAccessSIDs)
363+
<< MakeDropSecretObjectSql(secretName2);
319364
}
320365

321366
TString MakeDeleteExternalDataTableQuery(const TString& tableName) {

‎ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,10 @@ namespace NPrivate {
1111
TMaybe<TString> CreateSecretObjectQuery(const FederatedQuery::ConnectionSetting& setting,
1212
const TString& name,
1313
const TSigner::TPtr& signer,
14-
const TString& folderId);
14+
const TString& folderId,
15+
const TVector<TString>& externalSourcesAccessSIDs);
1516

16-
TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId);
17+
TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId, const TVector<TString>& externalSourcesAccessSIDs);
1718

1819
TString MakeCreateExternalDataSourceQuery(
1920
const FederatedQuery::ConnectionContent& connectionContent,

‎ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -564,7 +564,8 @@ IActor* MakeCreateConnectionActor(
564564
auto createSecretStatement = CreateSecretObjectQuery(connectionContent.setting(),
565565
connectionContent.name(),
566566
signer,
567-
folderId);
567+
folderId,
568+
computeConfig.GetExternalSourcesAccessSIDs(scope));
568569

569570
std::vector<TSchemaQueryTask> statements;
570571
if (createSecretStatement) {
@@ -665,22 +666,25 @@ IActor* MakeModifyConnectionActor(
665666
auto& newConnectionContent = request->Get()->Request.content();
666667
const auto& scope = request->Get()->Scope;
667668
const TString folderId = NYdb::NFq::TScope{scope}.ParseFolder();
669+
const auto externalSourcesAccessSIDs = computeConfig.GetExternalSourcesAccessSIDs(scope);
668670

669671
auto dropOldSecret =
670-
DropSecretObjectQuery(oldConnectionContent.name(), folderId);
672+
DropSecretObjectQuery(oldConnectionContent.name(), folderId, externalSourcesAccessSIDs);
671673
auto createNewSecret =
672674
CreateSecretObjectQuery(newConnectionContent.setting(),
673675
newConnectionContent.name(),
674676
signer,
675-
folderId);
677+
folderId,
678+
externalSourcesAccessSIDs);
676679

677680
bool replaceSupported = computeConfig.IsReplaceIfExistsSyntaxSupported();
678681
if (replaceSupported &&
679682
oldConnectionContent.name() == newConnectionContent.name()) {
680683
// CREATE OR REPLACE
681684
auto createSecretStatement =
682685
CreateSecretObjectQuery(newConnectionContent.setting(),
683-
newConnectionContent.name(), signer, folderId);
686+
newConnectionContent.name(), signer, folderId,
687+
externalSourcesAccessSIDs);
684688

685689
std::vector<TSchemaQueryTask> statements;
686690
if (createSecretStatement) {
@@ -727,13 +731,14 @@ IActor* MakeModifyConnectionActor(
727731
.SQL = *dropOldSecret,
728732
.RollbackSQL = CreateSecretObjectQuery(oldConnectionContent.setting(),
729733
oldConnectionContent.name(),
730-
signer, folderId),
734+
signer, folderId,
735+
externalSourcesAccessSIDs),
731736
.ShouldSkipStepOnError = IsPathDoesNotExistIssue});
732737
}
733738
if (createNewSecret) {
734739
statements.push_back(TSchemaQueryTask{.SQL = *createNewSecret,
735740
.RollbackSQL = DropSecretObjectQuery(
736-
newConnectionContent.name(), folderId)});
741+
newConnectionContent.name(), folderId, externalSourcesAccessSIDs)});
737742
}
738743

739744
statements.push_back(
@@ -787,18 +792,20 @@ IActor* MakeDeleteConnectionActor(
787792
TDuration requestTimeout,
788793
TCounters& counters,
789794
const TCommonConfig& commonConfig,
795+
const ::NFq::TComputeConfig& computeConfig,
790796
TSigner::TPtr signer) {
791797
auto queryFactoryMethod =
792798
[signer = std::move(signer),
793-
commonConfig](
799+
commonConfig, computeConfig](
794800
const TEvControlPlaneProxy::TEvDeleteConnectionRequest::TPtr& request)
795801
-> std::vector<TSchemaQueryTask> {
796802
auto& connectionContent = *request->Get()->ConnectionContent;
797803
const auto& scope = request->Get()->Scope;
798804
const TString folderId = NYdb::NFq::TScope{scope}.ParseFolder();
805+
const auto externalSourcesAccessSIDs = computeConfig.GetExternalSourcesAccessSIDs(scope);
799806

800807
auto dropSecret =
801-
DropSecretObjectQuery(connectionContent.name(), folderId);
808+
DropSecretObjectQuery(connectionContent.name(), folderId, externalSourcesAccessSIDs);
802809

803810
std::vector statements = {
804811
TSchemaQueryTask{.SQL = TString{MakeDeleteExternalDataSourceQuery(
@@ -812,7 +819,8 @@ IActor* MakeDeleteConnectionActor(
812819
.RollbackSQL =
813820
CreateSecretObjectQuery(connectionContent.setting(),
814821
connectionContent.name(),
815-
signer, folderId),
822+
signer, folderId,
823+
externalSourcesAccessSIDs),
816824
.ShouldSkipStepOnError = IsPathDoesNotExistIssue});
817825
}
818826
return statements;

‎ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ NActors::IActor* MakeDeleteConnectionActor(
4545
TDuration requestTimeout,
4646
TCounters& counters,
4747
const NConfig::TCommonConfig& commonConfig,
48+
const TComputeConfig& computeConfig,
4849
TSigner::TPtr signer);
4950

5051
/// Binding manipulation actors

‎ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1872,6 +1872,7 @@ class TControlPlaneProxyActor : public NActors::TActorBootstrapped<TControlPlane
18721872
Config.RequestTimeout,
18731873
Counters,
18741874
Config.CommonConfig,
1875+
Config.ComputeConfig,
18751876
Signer));
18761877
return;
18771878
}

0 commit comments

Comments
 (0)