Implement dual-config mode for distconf

ydb/core/blobstorage/nodewarden/distconf_generate.cpp

@@ -43,29 +43,44 @@ namespace NKikimr::NStorage {
 
         // initial config YAML is taken from the Cfg->SelfManagementConfig as it is cleared in TStorageConfig while
         // deriving it from NodeWarden configuration
-        if (!Cfg->SelfManagementConfig || !Cfg->SelfManagementConfig->HasInitialConfigYaml()) {
+        if (!Cfg->StartupConfigYaml) {
             return "missing initial config YAML";
         }
-        TStringStream ss;
-        TString yaml = Cfg->SelfManagementConfig->GetInitialConfigYaml();
-        ui32 version = 0;
+
+        ui64 version = 0;
         try {
-            auto json = NYaml::Yaml2Json(YAML::Load(yaml), true);
-            if (json.Has("metadata")) {
-                if (auto& metadata = json["metadata"]; metadata.Has("version")) {
-                    version = metadata["version"].GetUIntegerRobust();
-                }
-            }
+            version = NYamlConfig::GetMainMetadata(Cfg->StartupConfigYaml).Version.value_or(0);
         } catch (const std::exception& ex) {
-            return TStringBuilder() << "failed to parse initial config YAML: " << ex.what();
+            return TStringBuilder() << "failed to parse initial main YAML: " << ex.what();
         }
         if (version) {
-            return TStringBuilder() << "initial config version must be zero";
+            return TStringBuilder() << "initial main config version must be zero";
         }
-        if (const auto& error = UpdateConfigComposite(*config, yaml, std::nullopt)) {
+
+        if (const auto& error = UpdateConfigComposite(*config, Cfg->StartupConfigYaml, std::nullopt)) {
             return TStringBuilder() << "failed to update config yaml: " << *error;
         }
 
+        if (Cfg->StartupStorageYaml) {
+            ui64 storageVersion = 0;
+            try {
+                storageVersion = NYamlConfig::GetStorageMetadata(*Cfg->StartupStorageYaml).Version.value_or(0);
+            } catch (const std::exception& ex) {
+                return TStringBuilder() << "failed to parse initial storage YAML: " << ex.what();
+            }
+            if (storageVersion) {
+                return TStringBuilder() << "initial storage config version must be zero";
+            }
+
+            TString s;
+            if (TStringOutput output(s); true) {
+                TZstdCompress zstd(&output);
+                zstd << *Cfg->StartupStorageYaml;
+            }
+            config->SetCompressedStorageYaml(s);
+            config->SetExpectedStorageYamlVersion(storageVersion + 1);
+        }
+
         if (!Cfg->DomainsConfig) { // no automatic configuration required
         } else if (Cfg->DomainsConfig->StateStorageSize() == 1) { // the StateStorage config is already defined explicitly, just migrate it
             const auto& ss = Cfg->DomainsConfig->GetStateStorage(0);

ydb/core/blobstorage/nodewarden/distconf_invoke.cpp

@@ -665,11 +665,19 @@ namespace NKikimr::NStorage {
                 ? std::make_optional(request.GetSwitchDedicatedStorageSection())
                 : std::nullopt;
 
-            const TString *storageYamlPtr = newStorageYaml ? &newStorageYaml.value() :
-                Self->StorageConfigYaml ? &Self->StorageConfigYaml.value() : nullptr;
-
             const bool targetDedicatedStorageSection = switchDedicatedStorageSection.value_or(Self->StorageConfigYaml.has_value());
 
+            if (switchDedicatedStorageSection) {
+                // check that configs are explicitly defined when we are switching dual-config mode
+                if (!NewYaml) {
+                    return FinishWithError(TResult::ERROR, "main config must be specified when switching dedicated"
+                        " storage section mode");
+                } else if (*switchDedicatedStorageSection && !newStorageYaml) {
+                    return FinishWithError(TResult::ERROR, "storage config must be specified when turning on dedicated"
+                        " storage section mode");
+                }
+            }
+
             if (request.GetDedicatedStorageSectionConfigMode() != targetDedicatedStorageSection) {
                 return FinishWithError(TResult::ERROR, "DedicatedStorageSectionConfigMode does not match target state");
             } else if (newStorageYaml && !targetDedicatedStorageSection) {
@@ -678,89 +686,109 @@ namespace NKikimr::NStorage {
             } else if (switchDedicatedStorageSection && *switchDedicatedStorageSection == Self->StorageConfigYaml.has_value()) {
                 // this enable/disable command does not change the state
                 return FinishWithError(TResult::ERROR, "dedicated storage config section is already in requested state");
-            } else if (targetDedicatedStorageSection && !storageYamlPtr) {
-                // we are going to turn on dual-config mode, but no storage config provided
-                return FinishWithError(TResult::ERROR, "no dedicated storage config section provided");
             }
 
-            const TString *mainYamlPtr = NewYaml ? &NewYaml.value() : &Self->MainConfigYaml;
-
-            std::optional<ui64> newYamlVersion;
-            std::optional<ui64> newStorageYamlVersion;
+            TString state;
+            NKikimrBlobStorage::TStorageConfig config(*Self->StorageConfig);
+            std::optional<ui64> newExpectedStorageYamlVersion;
 
-            NKikimrConfig::TAppConfig appConfig;
-            const char *state = "";
+            if (config.HasExpectedStorageYamlVersion()) {
+                newExpectedStorageYamlVersion.emplace(config.GetExpectedStorageYamlVersion());
+            }
 
             try {
-                if (storageYamlPtr) { // parse the storage yaml first
-                    state = "loading storage YAML";
-                    auto json = NYaml::Yaml2Json(YAML::Load(*storageYamlPtr), true);
-                    state = "parsing storage YAML";
-                    NYaml::Parse(json, NYaml::GetJsonToProtoConfig(), appConfig, true);
-                    state = "extracting storage YAML metadata";
-                    if (json.Has("metadata")) {
-                        if (auto& metadata = json["metadata"]; metadata.Has("version")) {
-                            newStorageYamlVersion = metadata["version"].GetUIntegerRobust();
-                        }
+                auto load = [&](const TString& yaml, ui64& version, const char *expectedKind) {
+                    state = TStringBuilder() << "loading " << expectedKind << " YAML";
+                    NJson::TJsonValue json = NYaml::Yaml2Json(YAML::Load(yaml), true);
+
+                    state = TStringBuilder() << "extracting " << expectedKind << " metadata";
+                    if (!json.Has("metadata") || !json["metadata"].IsMap()) {
+                        throw yexception() << "no metadata section";
                     }
+                    auto& metadata = json["metadata"];
+                    NYaml::ValidateMetadata(metadata);
+                    if (!metadata.Has("kind") || metadata["kind"] != expectedKind) {
+                        throw yexception() << "missing or invalid kind provided";
+                    }
+                    version = metadata["version"].GetUIntegerRobust();
+
+                    state = TStringBuilder() << "validating " << expectedKind << " config section";
+                    if (!json.Has("config") || !json["config"].IsMap()) {
+                        throw yexception() << "missing config section";
+                    }
+
+                    return json;
+                };
+
+                NJson::TJsonValue main;
+                NJson::TJsonValue storage;
+                const NJson::TJsonValue *effective = nullptr;
+
+                if (newStorageYaml) {
+                    ui64 version = 0;
+                    storage = load(*newStorageYaml, version, "StorageConfig");
+                    if (const ui64 expected = Self->StorageConfig->GetExpectedStorageYamlVersion(); version != expected) {
+                        return FinishWithError(TResult::ERROR, TStringBuilder()
+                            << "storage config version must be increasing by one"
+                            << " new version# " << version
+                            << " expected version# " << expected);
+                    }
+
+                    newExpectedStorageYamlVersion = version + 1;
+                    effective = &storage;
                 }
 
-                state = "loading main YAML";
-                auto json = NYaml::Yaml2Json(YAML::Load(*mainYamlPtr), true);
-                state = "parsing main YAML";
-                NYaml::Parse(json, NYaml::GetJsonToProtoConfig(), appConfig, true);
-                state = "extracting main YAML metadata";
-                if (json.Has("metadata")) {
-                    if (auto& metadata = json["metadata"]; metadata.Has("version")) {
-                        newYamlVersion = metadata["version"].GetUIntegerRobust();
+                if (NewYaml) {
+                    ui64 version = 0;
+                    main = load(*NewYaml, version, "MainConfig");
+                    if (const ui64 expected = *Self->MainConfigYamlVersion + 1; version != expected) {
+                        return FinishWithError(TResult::ERROR, TStringBuilder()
+                            << "main config version must be increasing by one"
+                            << " new version# " << version
+                            << " expected version# " << expected);
+                    }
+
+                    if (!effective && !Self->StorageConfigYaml) {
+                        effective = &main;
                     }
                 }
-            } catch (const std::exception& ex) {
-                return FinishWithError(TResult::ERROR, TStringBuilder() << "exception while " << state
-                    << ": " << ex.what());
-            }
 
-            if (newYamlVersion && *newYamlVersion != *Self->MainConfigYamlVersion + 1) {
-                return FinishWithError(TResult::ERROR, TStringBuilder() << "version must be increasing by one"
-                    << " new version# " << *newYamlVersion << " expected version# " << *Self->MainConfigYamlVersion + 1);
-            } else if (newStorageYamlVersion && *newStorageYamlVersion != Self->StorageConfig->GetExpectedStorageYamlVersion()) {
-                return FinishWithError(TResult::ERROR, TStringBuilder() << "version must be increasing by one"
-                    << " new version# " << *newStorageYamlVersion
-                    << " expected version# " << Self->StorageConfig->GetExpectedStorageYamlVersion());
-            }
+                if (effective) {
+                    state = "parsing final config";
 
-            TString errorReason;
-            NKikimrBlobStorage::TStorageConfig config(*Self->StorageConfig);
-            const bool success = DeriveStorageConfig(appConfig, &config, &errorReason);
-            if (!success) {
-                return FinishWithError(TResult::ERROR, TStringBuilder() << "error while deriving StorageConfig: "
-                    << errorReason);
+                    NKikimrConfig::TAppConfig appConfig;
+                    NYaml::Parse(*effective, NYaml::GetJsonToProtoConfig(), appConfig, true);
+
+                    if (TString errorReason; !DeriveStorageConfig(appConfig, &config, &errorReason)) {
+                        return FinishWithError(TResult::ERROR, TStringBuilder()
+                            << "error while deriving StorageConfig: " << errorReason);
+                    }
+                }
+            } catch (const std::exception& ex) {
+                 return FinishWithError(TResult::ERROR, TStringBuilder() << "exception while " << state
+                    << ": " << ex.what());
             }
 
             if (NewYaml) {
                 if (const auto& error = UpdateConfigComposite(config, *NewYaml, std::nullopt)) {
                     return FinishWithError(TResult::ERROR, TStringBuilder() << "failed to update config yaml: " << *error);
                 }
-            } else {
-                config.SetConfigComposite(Self->StorageConfig->GetConfigComposite());
+            }
+
+            if (newExpectedStorageYamlVersion) {
+                config.SetExpectedStorageYamlVersion(*newExpectedStorageYamlVersion);
             }
 
             if (newStorageYaml) {
                 // make new compressed storage yaml section
                 TString s;
                 if (TStringOutput output(s); true) {
                     TZstdCompress zstd(&output);
-                    ::Save(&zstd, *newStorageYaml);
-                    ::Save(&zstd, *newStorageYamlVersion);
+                    zstd << *newStorageYaml;
                 }
                 config.SetCompressedStorageYaml(s);
-                config.SetExpectedStorageYamlVersion(*newStorageYamlVersion + 1);
-            } else if (switchDedicatedStorageSection && !*switchDedicatedStorageSection) {
-                // delete compressed storage yaml section as this request turns off dedicated storage yaml
-            } else if (Self->StorageConfig->HasCompressedStorageYaml()) {
-                // retain current storage yaml
-                config.SetCompressedStorageYaml(Self->StorageConfig->GetCompressedStorageYaml());
-                config.SetExpectedStorageYamlVersion(Self->StorageConfig->GetExpectedStorageYamlVersion());
+            } else if (!targetDedicatedStorageSection) {
+                config.ClearCompressedStorageYaml();
             }
 
             // advance the config generation
@@ -774,9 +802,7 @@ namespace NKikimr::NStorage {
                     << "ReplaceStorageConfig config validation failed: " << *error);
             }
 
-            const bool pushToConsole = true;
-
-            if (!pushToConsole || !request.GetSkipConsoleValidation()) {
+            if (request.GetSkipConsoleValidation() || !NewYaml) {
                 return StartProposition(&config);
             }
 
@@ -785,7 +811,7 @@ namespace NKikimr::NStorage {
                 !Self->SelfManagementEnabled &&
                 config.GetSelfManagementConfig().GetEnabled();
 
-            if (NewYaml && !Self->EnqueueConsoleConfigValidation(SelfId(), enablingDistconf, *NewYaml)) {
+            if (!Self->EnqueueConsoleConfigValidation(SelfId(), enablingDistconf, *NewYaml)) {
                 FinishWithError(TResult::ERROR, "console pipe is not available");
             } else {
                 ProposedStorageConfig = std::move(config);

ydb/core/blobstorage/nodewarden/distconf_mon.cpp

@@ -1,4 +1,5 @@
 #include "distconf.h"
+#include "node_warden_impl.h"
 
 #include <google/protobuf/util/json_util.h>
 
@@ -171,7 +172,9 @@ namespace NKikimr::NStorage {
                             if (config) {
                                 TString s;
                                 NProtoBuf::TextFormat::PrintToString(*config, &s);
-                                out << "<pre>" << s << "</pre>";
+                                out << "<pre>";
+                                EscapeHtmlString(out, s);
+                                out << "</pre>";
                             } else {
                                 out << "not defined";
                             }

ydb/core/blobstorage/nodewarden/node_warden.h

@@ -24,6 +24,8 @@ namespace NKikimr {
         std::optional<NKikimrConfig::TSelfManagementConfig> SelfManagementConfig;
         TString ConfigStorePath;
         std::optional<NKikimrBlobStorage::TYamlConfig> YamlConfig;
+        TString StartupConfigYaml;
+        std::optional<TString> StartupStorageYaml;
         TIntrusivePtr<IPDiskServiceFactory> PDiskServiceFactory;
         TIntrusivePtr<TAllVDiskKinds> AllVDiskKinds;
         TIntrusivePtr<NPDisk::TDriveModelDb> AllDriveModels;