Skip to content

Commit f693dfb

Browse files
alexvrualexvru
authored
Merge a7914d5 into b52dd35 
2 parents b52dd35 + a7914d5 commit f693dfb

17 files changed

+315
-153
lines changed

‎ydb/core/blobstorage/nodewarden/distconf_generate.cpp

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -43,29 +43,44 @@ namespace NKikimr::NStorage {
4343

4444
// initial config YAML is taken from the Cfg->SelfManagementConfig as it is cleared in TStorageConfig while
4545
// deriving it from NodeWarden configuration
46-
if (!Cfg->SelfManagementConfig || !Cfg->SelfManagementConfig->HasInitialConfigYaml()) {
46+
if (!Cfg->StartupConfigYaml) {
4747
return "missing initial config YAML";
4848
}
49-
TStringStream ss;
50-
TString yaml = Cfg->SelfManagementConfig->GetInitialConfigYaml();
51-
ui32 version = 0;
49+
50+
ui64 version = 0;
5251
try {
53-
auto json = NYaml::Yaml2Json(YAML::Load(yaml), true);
54-
if (json.Has("metadata")) {
55-
if (auto& metadata = json["metadata"]; metadata.Has("version")) {
56-
version = metadata["version"].GetUIntegerRobust();
57-
}
58-
}
52+
version = NYamlConfig::GetMainMetadata(Cfg->StartupConfigYaml).Version.value_or(0);
5953
} catch (const std::exception& ex) {
60-
return TStringBuilder() << "failed to parse initial config YAML: " << ex.what();
54+
return TStringBuilder() << "failed to parse initial main YAML: " << ex.what();
6155
}
6256
if (version) {
63-
return TStringBuilder() << "initial config version must be zero";
57+
return TStringBuilder() << "initial main config version must be zero";
6458
}
65-
if (const auto& error = UpdateConfigComposite(*config, yaml, std::nullopt)) {
59+
60+
if (const auto& error = UpdateConfigComposite(*config, Cfg->StartupConfigYaml, std::nullopt)) {
6661
return TStringBuilder() << "failed to update config yaml: " << *error;
6762
}
6863

64+
if (Cfg->StartupStorageYaml) {
65+
ui64 storageVersion = 0;
66+
try {
67+
storageVersion = NYamlConfig::GetStorageMetadata(*Cfg->StartupStorageYaml).Version.value_or(0);
68+
} catch (const std::exception& ex) {
69+
return TStringBuilder() << "failed to parse initial storage YAML: " << ex.what();
70+
}
71+
if (storageVersion) {
72+
return TStringBuilder() << "initial storage config version must be zero";
73+
}
74+
75+
TString s;
76+
if (TStringOutput output(s); true) {
77+
TZstdCompress zstd(&output);
78+
zstd << *Cfg->StartupStorageYaml;
79+
}
80+
config->SetCompressedStorageYaml(s);
81+
config->SetExpectedStorageYamlVersion(storageVersion + 1);
82+
}
83+
6984
if (!Cfg->DomainsConfig) { // no automatic configuration required
7085
} else if (Cfg->DomainsConfig->StateStorageSize() == 1) { // the StateStorage config is already defined explicitly, just migrate it
7186
const auto& ss = Cfg->DomainsConfig->GetStateStorage(0);

‎ydb/core/blobstorage/nodewarden/distconf_invoke.cpp

Lines changed: 45 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -689,42 +689,56 @@ namespace NKikimr::NStorage {
689689
std::optional<ui64> newStorageYamlVersion;
690690

691691
NKikimrConfig::TAppConfig appConfig;
692-
const char *state = "";
692+
TString state;
693693

694694
try {
695-
if (storageYamlPtr) { // parse the storage yaml first
696-
state = "loading storage YAML";
697-
auto json = NYaml::Yaml2Json(YAML::Load(*storageYamlPtr), true);
698-
state = "parsing storage YAML";
699-
NYaml::Parse(json, NYaml::GetJsonToProtoConfig(), appConfig, true);
700-
state = "extracting storage YAML metadata";
701-
if (json.Has("metadata")) {
702-
if (auto& metadata = json["metadata"]; metadata.Has("version")) {
703-
newStorageYamlVersion = metadata["version"].GetUIntegerRobust();
704-
}
695+
auto load = [&](const TString& yaml, std::optional<ui64>& version, const char *expectedKind) {
696+
state = TStringBuilder() << "loading " << expectedKind << " YAML";
697+
NJson::TJsonValue json = NYaml::Yaml2Json(YAML::Load(yaml), true);
698+
699+
state = TStringBuilder() << "extracting " << expectedKind << " metadata";
700+
if (!json.Has("metadata") || !json["metadata"].IsMap()) {
701+
throw yexception() << "no metadata section";
705702
}
706-
}
703+
auto& metadata = json["metadata"];
704+
NYaml::ValidateMetadata(metadata);
705+
if (!metadata.Has("kind") || metadata["kind"] != expectedKind) {
706+
throw yexception() << "missing or invalid kind provided";
707+
}
708+
version.emplace(metadata["version"].GetUIntegerRobust());
707709

708-
state = "loading main YAML";
709-
auto json = NYaml::Yaml2Json(YAML::Load(*mainYamlPtr), true);
710-
state = "parsing main YAML";
711-
NYaml::Parse(json, NYaml::GetJsonToProtoConfig(), appConfig, true);
712-
state = "extracting main YAML metadata";
713-
if (json.Has("metadata")) {
714-
if (auto& metadata = json["metadata"]; metadata.Has("version")) {
715-
newYamlVersion = metadata["version"].GetUIntegerRobust();
710+
state = TStringBuilder() << "validating " << expectedKind << " config section";
711+
if (!json.Has("config") || !json["config"].IsMap()) {
712+
throw yexception() << "missing config section";
716713
}
714+
715+
return json;
716+
};
717+
718+
NJson::TJsonValue main = load(*mainYamlPtr, newYamlVersion, "MainConfig");
719+
NJson::TJsonValue storage;
720+
const NJson::TJsonValue *effective;
721+
722+
if (storageYamlPtr) { // parse the storage yaml first
723+
storage = load(*storageYamlPtr, newStorageYamlVersion, "StorageConfig");
724+
effective = &storage;
725+
} else {
726+
effective = &main;
717727
}
728+
729+
state = "parsing final config";
730+
NYaml::Parse(*effective, NYaml::GetJsonToProtoConfig(), appConfig, true);
718731
} catch (const std::exception& ex) {
719-
return FinishWithError(TResult::ERROR, TStringBuilder() << "exception while " << state
732+
return FinishWithError(TResult::ERROR, TStringBuilder() << "exception while " << state
720733
<< ": " << ex.what());
721734
}
722735

723736
if (newYamlVersion && *newYamlVersion != *Self->MainConfigYamlVersion + 1) {
724-
return FinishWithError(TResult::ERROR, TStringBuilder() << "version must be increasing by one"
725-
<< " new version# " << *newYamlVersion << " expected version# " << *Self->MainConfigYamlVersion + 1);
737+
return FinishWithError(TResult::ERROR, TStringBuilder() << "main config version must be increasing by one"
738+
<< " new version# " << *newYamlVersion
739+
<< " expected version# " << *Self->MainConfigYamlVersion + 1);
726740
} else if (newStorageYamlVersion && *newStorageYamlVersion != Self->StorageConfig->GetExpectedStorageYamlVersion()) {
727-
return FinishWithError(TResult::ERROR, TStringBuilder() << "version must be increasing by one"
741+
return FinishWithError(TResult::ERROR, TStringBuilder() << "storage config version must be increasing by one"
728742
<< " new version# " << *newStorageYamlVersion
729743
<< " expected version# " << Self->StorageConfig->GetExpectedStorageYamlVersion());
730744
}
@@ -745,22 +759,22 @@ namespace NKikimr::NStorage {
745759
config.SetConfigComposite(Self->StorageConfig->GetConfigComposite());
746760
}
747761

762+
if (Self->StorageConfig->HasExpectedStorageYamlVersion()) {
763+
config.SetExpectedStorageYamlVersion(Self->StorageConfig->GetExpectedStorageYamlVersion());
764+
}
765+
748766
if (newStorageYaml) {
749767
// make new compressed storage yaml section
750768
TString s;
751769
if (TStringOutput output(s); true) {
752770
TZstdCompress zstd(&output);
753-
::Save(&zstd, *newStorageYaml);
754-
::Save(&zstd, *newStorageYamlVersion);
771+
zstd << *newStorageYaml;
755772
}
756773
config.SetCompressedStorageYaml(s);
757774
config.SetExpectedStorageYamlVersion(*newStorageYamlVersion + 1);
758-
} else if (switchDedicatedStorageSection && !*switchDedicatedStorageSection) {
759-
// delete compressed storage yaml section as this request turns off dedicated storage yaml
760-
} else if (Self->StorageConfig->HasCompressedStorageYaml()) {
761-
// retain current storage yaml
775+
} else if (targetDedicatedStorageSection) {
776+
Y_ABORT_UNLESS(Self->StorageConfig->HasCompressedStorageYaml()); // we must have one
762777
config.SetCompressedStorageYaml(Self->StorageConfig->GetCompressedStorageYaml());
763-
config.SetExpectedStorageYamlVersion(Self->StorageConfig->GetExpectedStorageYamlVersion());
764778
}
765779

766780
// advance the config generation

‎ydb/core/blobstorage/nodewarden/node_warden.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ namespace NKikimr {
2424
std::optional<NKikimrConfig::TSelfManagementConfig> SelfManagementConfig;
2525
TString ConfigStorePath;
2626
std::optional<NKikimrBlobStorage::TYamlConfig> YamlConfig;
27+
TString StartupConfigYaml;
28+
std::optional<TString> StartupStorageYaml;
2729
TIntrusivePtr<IPDiskServiceFactory> PDiskServiceFactory;
2830
TIntrusivePtr<TAllVDiskKinds> AllVDiskKinds;
2931
TIntrusivePtr<NPDisk::TDriveModelDb> AllDriveModels;

‎ydb/core/blobstorage/nodewarden/node_warden_impl.cpp

Lines changed: 67 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -443,8 +443,7 @@ void TNodeWarden::Bootstrap() {
443443

444444
//LoadConfigVersion();
445445
if (Cfg->YamlConfig) {
446-
YamlConfig.emplace();
447-
YamlConfig->CopyFrom(*Cfg->YamlConfig);
446+
YamlConfig.emplace(std::move(*Cfg->YamlConfig));
448447
}
449448

450449
// Start a statically configured set
@@ -635,31 +634,42 @@ void TNodeWarden::ProcessShredStatus(ui64 cookie, ui64 generation, std::optional
635634
}
636635
}
637636

638-
void TNodeWarden::PersistConfig(const TString& configYaml, ui64 version, std::optional<TString> storageYaml) {
637+
void TNodeWarden::PersistConfig(std::optional<TString> mainYaml, ui64 mainYamlVersion, std::optional<TString> storageYaml,
638+
std::optional<ui64> storageYamlVersion) {
639639
if (!Cfg->ConfigStorePath) {
640640
return;
641641
}
642642

643-
if (YamlConfig && version <= YamlConfig->GetConfigVersion()) {
644-
return; // some kind of a race
643+
const bool updateMain = mainYaml && (!YamlConfig || YamlConfig->GetMainConfigVersion() < mainYamlVersion);
644+
645+
const bool updateStorage = !storageYamlVersion || // delete storage config file in single-config mode
646+
storageYaml && (!YamlConfig || YamlConfig->GetStorageConfigVersion() < storageYamlVersion);
647+
648+
if (!updateMain && !updateStorage) {
649+
return; // nothing to do
645650
}
646651

647652
struct TSaveContext {
648653
TString ConfigStorePath;
649-
TString ConfigYaml;
650-
ui64 Version;
654+
std::optional<TString> MainYaml;
655+
ui64 MainYamlVersion;
651656
std::optional<TString> StorageYaml;
652-
bool Success = true;
653-
TString ErrorMessage;
654-
TActorId SelfId;
657+
std::optional<ui64> StorageYamlVersion;
658+
bool UpdateMain;
659+
bool UpdateStorage;
660+
bool DeleteStorage;
655661
};
656662

657-
auto saveCtx = std::make_shared<TSaveContext>();
658-
saveCtx->ConfigStorePath = Cfg->ConfigStorePath;
659-
saveCtx->ConfigYaml = std::move(configYaml);
660-
saveCtx->StorageYaml = std::move(storageYaml);
661-
saveCtx->Version = std::move(version);
662-
saveCtx->SelfId = SelfId();
663+
auto saveCtx = std::make_shared<TSaveContext>(TSaveContext{
664+
.ConfigStorePath = Cfg->ConfigStorePath,
665+
.MainYaml = std::move(mainYaml),
666+
.MainYamlVersion = mainYamlVersion,
667+
.StorageYaml = std::move(storageYaml),
668+
.StorageYamlVersion = storageYamlVersion,
669+
.UpdateMain = updateMain,
670+
.UpdateStorage = updateStorage,
671+
.DeleteStorage = !storageYamlVersion,
672+
});
663673

664674
EnqueueSyncOp([this, saveCtx](const TActorContext&) {
665675
bool success = true;
@@ -694,14 +704,16 @@ void TNodeWarden::PersistConfig(const TString& configYaml, ui64 version, std::op
694704
}
695705
};
696706

697-
if (success) {
698-
success = saveConfig(saveCtx->ConfigYaml, YamlConfigFileName);
707+
if (success && saveCtx->UpdateMain) {
708+
success = saveConfig(*saveCtx->MainYaml, YamlConfigFileName);
699709
if (success) {
700710
STLOG(PRI_INFO, BS_NODE, NW94, "Yaml config saved");
701711
}
702712
}
703713

704-
if (success && saveCtx->StorageYaml) {
714+
if (saveCtx->DeleteStorage) {
715+
std::filesystem::remove(std::filesystem::path(saveCtx->ConfigStorePath.c_str()) / StorageConfigFileName);
716+
} else if (success && saveCtx->UpdateStorage) {
705717
success = saveConfig(*saveCtx->StorageYaml, StorageConfigFileName);
706718
if (success) {
707719
STLOG(PRI_INFO, BS_NODE, NW95, "Storage config saved");
@@ -713,16 +725,22 @@ void TNodeWarden::PersistConfig(const TString& configYaml, ui64 version, std::op
713725
if (!YamlConfig) {
714726
YamlConfig.emplace();
715727
}
716-
YamlConfig->SetYAML(saveCtx->ConfigYaml);
717-
YamlConfig->SetConfigVersion(saveCtx->Version);
728+
if (saveCtx->UpdateMain) {
729+
YamlConfig->SetMainConfig(*saveCtx->MainYaml);
730+
YamlConfig->SetMainConfigVersion(saveCtx->MainYamlVersion);
731+
}
732+
if (saveCtx->DeleteStorage) {
733+
YamlConfig->ClearStorageConfig();
734+
YamlConfig->ClearStorageConfigVersion();
735+
} else if (saveCtx->UpdateStorage) {
736+
YamlConfig->SetStorageConfig(*saveCtx->StorageYaml);
737+
YamlConfig->SetStorageConfigVersion(*saveCtx->StorageYamlVersion);
738+
}
718739
ConfigSaveTimer.Reset();
719740
} else {
720-
NKikimrBlobStorage::TYamlConfig yamlConfig;
721-
yamlConfig.SetYAML(saveCtx->ConfigYaml);
722-
yamlConfig.SetConfigVersion(saveCtx->Version);
723-
TActivationContext::Schedule(TDuration::MilliSeconds(ConfigSaveTimer.NextBackoffMs()),
724-
new IEventHandle(SelfId(), SelfId(),
725-
new TEvPrivate::TEvRetrySaveConfig(yamlConfig), 0, ExpectedSaveConfigCookie));
741+
TActivationContext::Schedule(TDuration::MilliSeconds(ConfigSaveTimer.NextBackoffMs()), new IEventHandle(
742+
SelfId(), {}, new TEvPrivate::TEvRetrySaveConfig(std::move(saveCtx->MainYaml), saveCtx->MainYamlVersion,
743+
std::move(saveCtx->StorageYaml), saveCtx->StorageYamlVersion), 0, ExpectedSaveConfigCookie));
726744
}
727745
};
728746
});
@@ -733,7 +751,7 @@ void TNodeWarden::Handle(TEvRegisterPDiskLoadActor::TPtr ev) {
733751
}
734752

735753
void TNodeWarden::Handle(TEvBlobStorage::TEvControllerNodeServiceSetUpdate::TPtr ev) {
736-
const auto& record = ev->Get()->Record;
754+
auto& record = ev->Get()->Record;
737755

738756
if (record.HasAvailDomain() && record.GetAvailDomain() != AvailDomainId) {
739757
// AvailDomain may arrive unset
@@ -814,13 +832,26 @@ void TNodeWarden::Handle(TEvBlobStorage::TEvControllerNodeServiceSetUpdate::TPtr
814832
}
815833

816834
if (record.HasYamlConfig()) {
817-
const auto& request = record.GetYamlConfig();
818-
if (request.HasYAML()) {
819-
TString yaml = NYamlConfig::DecompressYamlString(request.GetYAML());
820-
ui64 version = request.GetConfigVersion();
821-
PersistConfig(yaml, version);
822-
ExpectedSaveConfigCookie++;
835+
auto& yaml = *record.MutableYamlConfig();
836+
837+
if (yaml.HasCompressedMainConfig()) {
838+
Y_DEBUG_ABORT_UNLESS(!yaml.HasMainConfig());
839+
yaml.SetMainConfig(NYamlConfig::DecompressYamlString(yaml.GetCompressedMainConfig()));
840+
yaml.ClearCompressedMainConfig();
841+
}
842+
843+
if (yaml.HasCompressedStorageConfig()) {
844+
Y_DEBUG_ABORT_UNLESS(!yaml.HasStorageConfig());
845+
yaml.SetStorageConfig(NYamlConfig::DecompressYamlString(yaml.GetCompressedStorageConfig()));
846+
yaml.ClearCompressedStorageConfig();
823847
}
848+
849+
PersistConfig(yaml.HasMainConfig() ? std::make_optional(yaml.GetMainConfig()) : std::nullopt,
850+
yaml.GetMainConfigVersion(),
851+
yaml.HasStorageConfig() ? std::make_optional(yaml.GetStorageConfig()) : std::nullopt,
852+
yaml.HasStorageConfigVersion() ? std::make_optional(yaml.GetStorageConfigVersion()) : std::nullopt);
853+
854+
ExpectedSaveConfigCookie++;
824855
}
825856
}
826857

@@ -1048,8 +1079,8 @@ void TNodeWarden::Handle(TEvPrivate::TEvUpdateNodeDrives::TPtr&) {
10481079
void TNodeWarden::Handle(TEvPrivate::TEvRetrySaveConfig::TPtr& ev) {
10491080
STLOG(PRI_TRACE, BS_NODE, NW97, "Handle(TEvRetrySaveConfig)");
10501081
if (ev->Cookie == ExpectedSaveConfigCookie) {
1051-
const auto& yamlConfig = ev->Get()->YamlConfig;
1052-
PersistConfig(yamlConfig.GetYAML(), yamlConfig.GetConfigVersion());
1082+
auto *msg = ev->Get();
1083+
PersistConfig(std::move(msg->MainYaml), msg->MainYamlVersion, std::move(msg->StorageYaml), msg->StorageYamlVersion);
10531084
ExpectedSaveConfigCookie++;
10541085
}
10551086
}
@@ -1270,7 +1301,6 @@ bool NKikimr::NStorage::DeriveStorageConfig(const NKikimrConfig::TAppConfig& app
12701301
return false;
12711302
}
12721303
smTo->CopyFrom(smFrom);
1273-
smTo->ClearInitialConfigYaml(); // do not let this section into final StorageConfig
12741304
} else {
12751305
config->ClearSelfManagementConfig();
12761306
}

‎ydb/core/blobstorage/nodewarden/node_warden_impl.h

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -177,9 +177,17 @@ namespace NKikimr::NStorage {
177177
};
178178

179179
struct TEvRetrySaveConfig : TEventLocal<TEvRetrySaveConfig, EvRetrySaveConfig> {
180-
NKikimrBlobStorage::TYamlConfig YamlConfig;
181-
TEvRetrySaveConfig(const NKikimrBlobStorage::TYamlConfig& yamlConfig)
182-
: YamlConfig(yamlConfig)
180+
std::optional<TString> MainYaml;
181+
ui64 MainYamlVersion;
182+
std::optional<TString> StorageYaml;
183+
std::optional<ui64> StorageYamlVersion;
184+
185+
TEvRetrySaveConfig(std::optional<TString> mainYaml, ui64 mainYamlVersion, std::optional<TString> storageYaml,
186+
std::optional<ui64> storageYamlVersion)
187+
: MainYaml(std::move(mainYaml))
188+
, MainYamlVersion(mainYamlVersion)
189+
, StorageYaml(std::move(storageYaml))
190+
, StorageYamlVersion(storageYamlVersion)
183191
{}
184192
};
185193
};
@@ -567,7 +575,8 @@ namespace NKikimr::NStorage {
567575
void Handle(NPDisk::TEvShredPDisk::TPtr ev);
568576
void ProcessShredStatus(ui64 cookie, ui64 generation, std::optional<TString> error);
569577

570-
void PersistConfig(const TString& yaml, ui64 version, std::optional<TString> storageYaml = std::nullopt);
578+
void PersistConfig(std::optional<TString> mainYaml, ui64 mainYamlVersion, std::optional<TString> storageYaml,
579+
std::optional<ui64> storageYamlVersion);
571580
void LoadConfigVersion();
572581

573582
void Handle(TEvRegisterPDiskLoadActor::TPtr ev);

0 commit comments

Comments
 (0)