Skip to content

Commit e13dad4

Browse files
alexvruEnjection
alexvru
authored and
Enjection
committed
Add EnableConfigV2 option to allow enabling/disabling v2 configuration for migration purposes (ydb-platform#16066)
1 parent 5b416ec commit e13dad4

File tree

14 files changed

+139
-41
lines changed

14 files changed

+139
-41
lines changed

‎ydb/core/blobstorage/base/blobstorage_console_events.h

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -77,14 +77,9 @@ namespace NKikimr {
7777
NKikimrBlobStorage::TEvControllerReplaceConfigRequest, EvControllerReplaceConfigRequest> {
7878
TEvControllerReplaceConfigRequest() = default;
7979

80-
TEvControllerReplaceConfigRequest(
81-
std::optional<TString> clusterYaml,
82-
std::optional<TString> storageYaml,
83-
std::optional<bool> switchDedicatedStorageSection,
84-
bool dedicatedConfigMode,
85-
bool allowUnknownFields,
86-
bool bypassMetadataChecks) {
87-
80+
TEvControllerReplaceConfigRequest(std::optional<TString> clusterYaml, std::optional<TString> storageYaml,
81+
std::optional<bool> switchDedicatedStorageSection, bool dedicatedConfigMode, bool allowUnknownFields,
82+
bool bypassMetadataChecks, bool enableConfigV2, bool disableConfigV2) {
8883
if (clusterYaml) {
8984
Record.SetClusterYaml(*clusterYaml);
9085
}
@@ -97,6 +92,11 @@ namespace NKikimr {
9792
Record.SetDedicatedConfigMode(dedicatedConfigMode);
9893
Record.SetAllowUnknownFields(allowUnknownFields);
9994
Record.SetBypassMetadataChecks(bypassMetadataChecks);
95+
if (enableConfigV2) {
96+
Record.SetSwitchEnableConfigV2(true);
97+
} else if (disableConfigV2) {
98+
Record.SetSwitchEnableConfigV2(false);
99+
}
100100
}
101101

102102
TString ToString() const override {

‎ydb/core/grpc_services/rpc_config.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,9 @@ class TReplaceStorageConfigRequest : public TBSConfigRequestGrpc<TReplaceStorage
232232
shim.SwitchDedicatedStorageSection,
233233
shim.DedicatedConfigMode,
234234
request->allow_unknown_fields() || request->bypass_checks(),
235-
request->bypass_checks());
235+
request->bypass_checks(),
236+
false /* TODO: implement */,
237+
false /* TODO: implement */);
236238
}
237239

238240
private:

‎ydb/core/grpc_services/rpc_config_base.h

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,14 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActor<TDerived, TRequest
284284
if constexpr (std::is_same_v<TResultRecord, Ydb::Config::FetchConfigResult>) {
285285
TResultRecord result;
286286
const auto& record = ev->Get()->Record;
287+
if (record.HasErrorReason()) {
288+
const auto status = record.GetDisabledConfigV2()
289+
? Ydb::StatusIds::UNSUPPORTED
290+
: Ydb::StatusIds::INTERNAL_ERROR;
291+
self->Reply(status, TStringBuilder() << "failed to fetch config: " << record.GetErrorReason(),
292+
NKikimrIssues::TIssuesIds::DEFAULT_ERROR, self->ActorContext());
293+
return;
294+
}
287295
if (record.HasClusterYaml()) {
288296
auto conf = ev->Get()->Record.GetClusterYaml();
289297
auto metadata = NYamlConfig::GetMainMetadata(conf);
@@ -327,7 +335,10 @@ class TBSConfigRequestGrpc : public TRpcOperationRequestActor<TDerived, TRequest
327335
TResultRecord result;
328336
self->ReplyWithResult(Ydb::StatusIds::SUCCESS, result, self->ActorContext());
329337
} else {
330-
self->Reply(Ydb::StatusIds::INTERNAL_ERROR, TStringBuilder() << "failed to replace configuration: "
338+
const auto status = record.GetDisabledConfigV2()
339+
? Ydb::StatusIds::UNSUPPORTED
340+
: Ydb::StatusIds::INTERNAL_ERROR;
341+
self->Reply(status, TStringBuilder() << "failed to replace configuration: "
331342
<< NKikimrBlobStorage::TEvControllerReplaceConfigResponse::EStatus_Name(record.GetStatus())
332343
<< ": " << record.GetErrorReason(), NKikimrIssues::TIssuesIds::DEFAULT_ERROR, self->ActorContext());
333344
}

‎ydb/core/mind/bscontroller/bsc.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ void TBlobStorageController::Handle(TEvBlobStorage::TEvControllerDistconfRequest
442442

443443
// commit it
444444
Execute(CreateTxCommitConfig(std::move(yamlConfig), std::make_optional(std::move(storageYaml)), std::nullopt,
445-
expectedStorageYamlConfigVersion, std::exchange(h, {})));
445+
expectedStorageYamlConfigVersion, std::exchange(h, {}), std::nullopt));
446446
break;
447447
}
448448

‎ydb/core/mind/bscontroller/commit_config.cpp

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ namespace NKikimr::NBsController {
1515
std::optional<NKikimrBlobStorage::TStorageConfig> StorageConfig;
1616
std::optional<ui64> ExpectedStorageYamlConfigVersion;
1717
std::unique_ptr<IEventHandle> Handle;
18+
std::optional<bool> SwitchEnableConfigV2;
1819

1920
ui64 GenerationOnStart = 0;
2021
TString FingerprintOnStart;
@@ -23,13 +24,15 @@ namespace NKikimr::NBsController {
2324
TTxCommitConfig(TBlobStorageController *controller, std::optional<TYamlConfig>&& yamlConfig,
2425
std::optional<std::optional<TString>>&& storageYamlConfig,
2526
std::optional<NKikimrBlobStorage::TStorageConfig>&& storageConfig,
26-
std::optional<ui64> expectedStorageYamlConfigVersion, std::unique_ptr<IEventHandle> handle)
27+
std::optional<ui64> expectedStorageYamlConfigVersion, std::unique_ptr<IEventHandle> handle,
28+
std::optional<bool> switchEnableConfigV2)
2729
: TTransactionBase(controller)
2830
, YamlConfig(std::move(yamlConfig))
2931
, StorageYamlConfig(std::move(storageYamlConfig))
3032
, StorageConfig(std::move(storageConfig))
3133
, ExpectedStorageYamlConfigVersion(expectedStorageYamlConfigVersion)
3234
, Handle(std::move(handle))
35+
, SwitchEnableConfigV2(switchEnableConfigV2)
3336
{}
3437

3538
TTxType GetTxType() const override { return NBlobStorageController::TXTYPE_COMMIT_CONFIG; }
@@ -53,6 +56,9 @@ namespace NKikimr::NBsController {
5356
if (ExpectedStorageYamlConfigVersion) {
5457
row.Update<Schema::State::ExpectedStorageYamlConfigVersion>(*ExpectedStorageYamlConfigVersion);
5558
}
59+
if (SwitchEnableConfigV2) {
60+
row.Update<Schema::State::EnableConfigV2>(*SwitchEnableConfigV2);
61+
}
5662
return true;
5763
}
5864

@@ -104,6 +110,9 @@ namespace NKikimr::NBsController {
104110
if (update && Self->StorageYamlConfig) {
105111
update->SetStorageConfigVersion(NYamlConfig::GetStorageMetadata(*Self->StorageYamlConfig).Version.value_or(0));
106112
}
113+
if (SwitchEnableConfigV2) {
114+
Self->EnableConfigV2 = *SwitchEnableConfigV2;
115+
}
107116

108117
if (Handle) {
109118
TActivationContext::Send(Handle.release());
@@ -126,9 +135,10 @@ namespace NKikimr::NBsController {
126135
ITransaction* TBlobStorageController::CreateTxCommitConfig(std::optional<TYamlConfig>&& yamlConfig,
127136
std::optional<std::optional<TString>>&& storageYamlConfig,
128137
std::optional<NKikimrBlobStorage::TStorageConfig>&& storageConfig,
129-
std::optional<ui64> expectedStorageYamlConfigVersion, std::unique_ptr<IEventHandle> handle) {
138+
std::optional<ui64> expectedStorageYamlConfigVersion, std::unique_ptr<IEventHandle> handle,
139+
std::optional<bool> switchEnableConfigV2) {
130140
return new TTxCommitConfig(this, std::move(yamlConfig), std::move(storageYamlConfig), std::move(storageConfig),
131-
expectedStorageYamlConfigVersion, std::move(handle));
141+
expectedStorageYamlConfigVersion, std::move(handle), switchEnableConfigV2);
132142
}
133143

134144
} // namespace NKikimr::NBsController

‎ydb/core/mind/bscontroller/console_interaction.cpp

Lines changed: 60 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,10 @@ namespace NKikimr::NBsController {
7575
auto& record = ev->Get()->Record;
7676
STLOG(PRI_DEBUG, BS_CONTROLLER, BSC19, "Console proposed config response", (Response, record));
7777

78+
if (!Self.EnableConfigV2 && !PendingReplaceRequest) {
79+
return;
80+
}
81+
7882
auto overwriteConfig = [&] {
7983
TString yamlReturnedByFetch = record.GetYAML();
8084
if (!yamlReturnedByFetch) {
@@ -101,7 +105,7 @@ namespace NKikimr::NBsController {
101105

102106
TYamlConfig yamlConfig(std::move(yaml), version, std::move(yamlReturnedByFetch));
103107
Self.Execute(Self.CreateTxCommitConfig(std::move(yamlConfig), std::nullopt,
104-
std::move(storageConfig), std::nullopt, nullptr));
108+
std::move(storageConfig), std::nullopt, nullptr, std::nullopt));
105109
CommitInProgress = true;
106110
}
107111
} catch (const std::exception& ex) {
@@ -146,14 +150,17 @@ namespace NKikimr::NBsController {
146150
break;
147151

148152
case NKikimrBlobStorage::TEvControllerProposeConfigResponse::ReverseCommit:
149-
if (!Self.YamlConfig && !Self.StorageYamlConfig) {
153+
if (PendingReplaceRequest) {
154+
ExpectedYamlConfigVersion.emplace(record.GetConsoleConfigVersion());
155+
Handle(PendingReplaceRequest);
156+
PendingReplaceRequest.Reset();
157+
} else if (!Self.YamlConfig && !Self.StorageYamlConfig) {
150158
overwriteConfig();
151159
} else {
152160
STLOG(PRI_CRIT, BS_CONTROLLER, BSC29, "ReverseCommit status received when BSC has YamlConfig/StorageYamlConfig",
153161
(YamlConfig, Self.YamlConfig), (StorageYamlConfig, Self.StorageYamlConfig), (Record, record));
154162
Y_DEBUG_ABORT();
155163
}
156-
157164
break;
158165

159166
default:
@@ -217,10 +224,15 @@ namespace NKikimr::NBsController {
217224

218225
auto& record = ev->Get()->Record;
219226

220-
if (!Working || CommitInProgress || ClientId) {
227+
const bool reasonOngoingCommit = CommitInProgress || (ClientId && ClientId != ev->Sender);
228+
if (reasonOngoingCommit || (!Self.EnableConfigV2 && !record.GetSwitchEnableConfigV2())) {
221229
// reply to newly came query
222230
const TActorId temp = std::exchange(ClientId, ev->Sender);
223-
IssueGRpcResponse(NKikimrBlobStorage::TEvControllerReplaceConfigResponse::OngoingCommit, "ongoing commit");
231+
if (reasonOngoingCommit) {
232+
IssueGRpcResponse(NKikimrBlobStorage::TEvControllerReplaceConfigResponse::OngoingCommit, "ongoing commit");
233+
} else {
234+
IssueGRpcResponse(NKikimrBlobStorage::TEvControllerReplaceConfigResponse::InvalidRequest, "configuration v2 is disabled", true);
235+
}
224236
ClientId = temp;
225237
return;
226238
}
@@ -233,6 +245,34 @@ namespace NKikimr::NBsController {
233245
"configuration is locked by distconf");
234246
}
235247

248+
SwitchEnableConfigV2 = record.HasSwitchEnableConfigV2()
249+
? std::make_optional(record.GetSwitchEnableConfigV2())
250+
: std::nullopt;
251+
252+
if (!Self.EnableConfigV2 && record.HasSwitchDedicatedStorageSection()) {
253+
return IssueGRpcResponse(NKikimrBlobStorage::TEvControllerReplaceConfigResponse::InvalidRequest,
254+
"can't enable configuration v2 and switch dedicated storage section mode at the same time");
255+
}
256+
257+
if (Self.EnableConfigV2 && SwitchEnableConfigV2 && !*SwitchEnableConfigV2 && Self.StorageYamlConfig) {
258+
return IssueGRpcResponse(NKikimrBlobStorage::TEvControllerReplaceConfigResponse::InvalidRequest,
259+
"can't revert to configuration v1 with dedicated storage section enabled");
260+
}
261+
262+
if (!Self.EnableConfigV2 && !PendingReplaceRequest) {
263+
Y_ABORT_UNLESS(SwitchEnableConfigV2);
264+
Y_ABORT_UNLESS(*SwitchEnableConfigV2);
265+
if (!ConsolePipe) {
266+
return IssueGRpcResponse(NKikimrBlobStorage::TEvControllerReplaceConfigResponse::SessionClosed,
267+
"connection to Console tablet terminated");
268+
}
269+
270+
// just ask console for the latest config
271+
NTabletPipe::SendData(Self.SelfId(), ConsolePipe, new TEvBlobStorage::TEvControllerProposeConfigRequest);
272+
PendingReplaceRequest = std::move(ev);
273+
return;
274+
}
275+
236276
PendingStorageYamlConfig.reset();
237277

238278
if (Self.StorageYamlConfig.has_value()) { // separate configuration
@@ -290,9 +330,7 @@ namespace NKikimr::NBsController {
290330
}
291331

292332
if (PendingYamlConfig) {
293-
const ui64 expected = Self.YamlConfig
294-
? GetVersion(*Self.YamlConfig) + 1
295-
: 0;
333+
const ui64 expected = ExpectedYamlConfigVersion.value_or(Self.YamlConfig ? GetVersion(*Self.YamlConfig) + 1 : 0);
296334

297335
if (!NYamlConfig::IsMainConfig(*PendingYamlConfig)) {
298336
return IssueGRpcResponse(NKikimrBlobStorage::TEvControllerReplaceConfigResponse::InvalidRequest,
@@ -342,7 +380,10 @@ namespace NKikimr::NBsController {
342380
void TBlobStorageController::TConsoleInteraction::Handle(TEvBlobStorage::TEvControllerFetchConfigRequest::TPtr &ev) {
343381
const auto& record = ev->Get()->Record;
344382
auto response = std::make_unique<TEvBlobStorage::TEvControllerFetchConfigResponse>();
345-
if (!Self.ConfigLock.empty() || Self.SelfManagementEnabled) {
383+
if (!Self.EnableConfigV2) {
384+
response->Record.SetErrorReason("configuration v2 is disabled");
385+
response->Record.SetDisabledConfigV2(true);
386+
} else if (!Self.ConfigLock.empty() || Self.SelfManagementEnabled) {
346387
response->Record.SetErrorReason("configuration is locked by distconf");
347388
} else if (Self.StorageYamlConfig) {
348389
if (record.GetDedicatedStorageSection()) {
@@ -455,7 +496,7 @@ namespace NKikimr::NBsController {
455496
}
456497

457498
Self.Execute(Self.CreateTxCommitConfig(std::move(yamlConfig), std::exchange(PendingStorageYamlConfig, {}),
458-
std::move(storageConfig), expectedStorageYamlConfigVersion, nullptr));
499+
std::move(storageConfig), expectedStorageYamlConfigVersion, nullptr, SwitchEnableConfigV2));
459500
CommitInProgress = true;
460501
PendingYamlConfig.reset();
461502
} catch (const TExError& error) {
@@ -498,13 +539,20 @@ namespace NKikimr::NBsController {
498539
}
499540

500541
void TBlobStorageController::TConsoleInteraction::IssueGRpcResponse(
501-
NKikimrBlobStorage::TEvControllerReplaceConfigResponse::EStatus status, std::optional<TString> errorReason) {
542+
NKikimrBlobStorage::TEvControllerReplaceConfigResponse::EStatus status, std::optional<TString> errorReason,
543+
bool disabledConfigV2) {
502544
Y_ABORT_UNLESS(ClientId);
503-
Self.Send(ClientId, new TEvBlobStorage::TEvControllerReplaceConfigResponse(status, std::move(errorReason)));
545+
auto resp = std::make_unique<TEvBlobStorage::TEvControllerReplaceConfigResponse>(status, std::move(errorReason));
546+
if (disabledConfigV2) {
547+
resp->Record.SetDisabledConfigV2(true);
548+
}
549+
Self.Send(ClientId, resp.release());
504550
ClientId = {};
505551
++ExpectedValidationTimeoutCookie; // spoil validation cookie as incoming GRPC request has expired
506552
PendingYamlConfig.reset();
507553
PendingStorageYamlConfig.reset();
554+
ExpectedYamlConfigVersion.reset();
555+
PendingReplaceRequest.Reset();
508556
}
509557

510558
}

‎ydb/core/mind/bscontroller/console_interaction.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,18 +47,20 @@ namespace NKikimr::NBsController {
4747
bool NeedRetrySession = false;
4848
bool Working = false;
4949
bool CommitInProgress = false;
50+
std::optional<bool> SwitchEnableConfigV2;
51+
TEvBlobStorage::TEvControllerReplaceConfigRequest::TPtr PendingReplaceRequest;
5052

5153
std::optional<TString> PendingYamlConfig;
5254
bool AllowUnknownFields = false;
53-
5455
std::optional<std::optional<TString>> PendingStorageYamlConfig;
56+
std::optional<ui64> ExpectedYamlConfigVersion;
5557

5658
void MakeCommitToConsole(TString& config, ui32 configVersion);
5759
void MakeGetBlock();
5860
void MakeRetrySession();
5961

6062
void IssueGRpcResponse(NKikimrBlobStorage::TEvControllerReplaceConfigResponse::EStatus status,
61-
std::optional<TString> errorReason = std::nullopt);
63+
std::optional<TString> errorReason = std::nullopt, bool disabledConfigV2 = false);
6264
};
6365

6466
}

‎ydb/core/mind/bscontroller/impl.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1562,6 +1562,7 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
15621562
bool AllowMultipleRealmsOccupation = true;
15631563
bool StorageConfigObtained = false;
15641564
bool Loaded = false;
1565+
bool EnableConfigV2 = false;
15651566
std::shared_ptr<TControlWrapper> EnableSelfHealWithDegraded;
15661567

15671568
struct TLifetimeToken {};
@@ -1980,7 +1981,8 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
19801981
std::optional<std::optional<TString>>&& storageYamlConfig,
19811982
std::optional<NKikimrBlobStorage::TStorageConfig>&& storageConfig,
19821983
std::optional<ui64> expectedStorageYamlConfigVersion,
1983-
std::unique_ptr<IEventHandle> handle);
1984+
std::unique_ptr<IEventHandle> handle,
1985+
std::optional<bool> switchEnableConfigV2);
19841986

19851987
struct TVDiskAvailabilityTiming {
19861988
TVSlotId VSlotId;

‎ydb/core/mind/bscontroller/load_everything.cpp

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ class TBlobStorageController::TTxLoadEverything : public TTransactionBase<TBlobS
109109
if (state.HaveValue<T::ShredState>()) {
110110
Self->ShredState.OnLoad(state.GetValue<T::ShredState>());
111111
}
112+
Self->EnableConfigV2 = state.GetValue<T::EnableConfigV2>();
112113
}
113114
}
114115

@@ -472,14 +473,6 @@ class TBlobStorageController::TTxLoadEverything : public TTransactionBase<TBlobS
472473
}
473474
}
474475

475-
// primitive garbage collection for obsolete metrics
476-
for (const auto& key : pdiskMetricsToDelete) {
477-
db.Table<Schema::PDiskMetrics>().Key(key).Delete();
478-
}
479-
for (const auto& key : vdiskMetricsToDelete) {
480-
db.Table<Schema::VDiskMetrics>().Key(key).Delete();
481-
}
482-
483476
// apply storage pool stats
484477
std::unordered_map<TBoxStoragePoolId, ui64> allocatedSizeMap;
485478
for (const auto& [vslotId, slot] : Self->VSlots) {
@@ -523,6 +516,14 @@ class TBlobStorageController::TTxLoadEverything : public TTransactionBase<TBlobS
523516
group->CalculateGroupStatus();
524517
}
525518

519+
// primitive garbage collection for obsolete metrics
520+
for (const auto& key : pdiskMetricsToDelete) {
521+
db.Table<Schema::PDiskMetrics>().Key(key).Delete();
522+
}
523+
for (const auto& key : vdiskMetricsToDelete) {
524+
db.Table<Schema::VDiskMetrics>().Key(key).Delete();
525+
}
526+
526527
return true;
527528
}
528529

‎ydb/core/mind/bscontroller/migrate.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,14 @@ class TBlobStorageController::TTxMigrate : public TTransactionBase<TBlobStorageC
171171
}
172172
};
173173

174+
class TTxUpdateEnableConfigV2 : public TTxBase {
175+
public:
176+
bool Execute(TTransactionContext& txc) override {
177+
NIceDb::TNiceDb(txc.DB).Table<Schema::State>().Key(true).Update<Schema::State::EnableConfigV2>(true);
178+
return true;
179+
}
180+
};
181+
174182
TDeque<THolder<TTxBase>> Queue;
175183

176184
public:
@@ -232,6 +240,10 @@ class TBlobStorageController::TTxMigrate : public TTransactionBase<TBlobStorageC
232240

233241
Queue.emplace_back(new TTxUpdateCompatibilityInfo);
234242

243+
if (!hasInstanceId) {
244+
Queue.emplace_back(new TTxUpdateEnableConfigV2);
245+
}
246+
235247
return true;
236248
}
237249

0 commit comments

Comments
 (0)