Refactor config transformations

ydb/core/driver_lib/cli_utils/cli_cmds_config.cpp

@@ -1,6 +1,6 @@
 #include <ydb/core/protos/blobstorage.pb.h>
 #include <ydb/core/protos/config.pb.h>
-#include <ydb/library/yaml_config/deprecated/yaml_config_parser.h>
+#include <ydb/library/yaml_config/yaml_config_parser.h>
 #include "cli.h"
 #include "cli_cmds.h"
 #include "proto_common.h"

ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp

@@ -4,7 +4,7 @@
 #include <ydb/core/base/path.h>
 #include <ydb/library/yaml_config/yaml_config.h>
 #include <ydb/core/driver_lib/run/run.h>
-#include <ydb/library/yaml_config/deprecated/yaml_config_parser.h>
+#include <ydb/library/yaml_config/yaml_config_parser.h>
 #include <ydb/public/lib/deprecated/kicli/kicli.h>
 #include <util/digest/city.h>
 #include <util/random/random.h>
@@ -827,8 +827,7 @@ class TClientCommandServerBase : public TClientCommand {
             return;
         }
         auto yamlConfig = TFileInput(YamlConfigFile);
-        NKikimrConfig::TAppConfig parsedConfig;
-        NKikimr::NYaml::Parse(yamlConfig.ReadAll(), parsedConfig);
+        NKikimrConfig::TAppConfig parsedConfig = NKikimr::NYaml::Parse(yamlConfig.ReadAll());
         const google::protobuf::Descriptor* descriptor = AppConfig.GetDescriptor();
         const google::protobuf::Reflection* reflection = AppConfig.GetReflection();
         for(int fieldIdx = 0; fieldIdx < descriptor->field_count(); ++fieldIdx) {

ydb/core/driver_lib/cli_utils/ya.make

@@ -52,7 +52,6 @@ PEERDIR(
     ydb/library/aclib
     ydb/library/folder_service/proto
     ydb/library/yaml_config
-    ydb/library/yaml_config/deprecated
     ydb/public/api/grpc
     ydb/public/api/grpc/draft
     ydb/public/lib/deprecated/client

ydb/library/yaml_config/core_constants.h

@@ -0,0 +1,66 @@
+#pragma once
+
+#include <util/generic/string.h>
+
+#include <utility>
+#include <array>
+#include <map>
+
+namespace NKikimr::NYaml {
+
+constexpr ui32 DEFAULT_INTERCONNECT_PORT = 19001;
+
+constexpr inline TStringBuf DEFAULT_ROOT_USERNAME = "root";
+
+constexpr inline TStringBuf COMBINED_DISK_INFO_PATH = "/blob_storage_config/service_set/groups/*/rings/*/fail_domains/*/vdisk_locations/*";
+constexpr inline TStringBuf GROUP_PATH = "/blob_storage_config/service_set/groups/*";
+constexpr inline TStringBuf DISABLE_BUILTIN_SECURITY_PATH = "/domains_config/disable_builtin_security";
+constexpr inline TStringBuf DEFAULT_GROUPS_PATH = "/domains_config/default_groups";
+constexpr inline TStringBuf DEFAULT_ACCESS_PATH = "/domains_config/default_access";
+
+constexpr inline TStringBuf ERASURE_SPECIES_FIELD = "erasure_species";
+
+const inline std::array<std::pair<TString, ui32>, 10> DEFAULT_TABLETS{
+    std::pair{TString{"FlatHive"}, 1},
+    std::pair{TString{"FlatBsController"}, 1},
+    std::pair{TString{"FlatSchemeshard"}, 1},
+    std::pair{TString{"FlatTxCoordinator"}, 3},
+    std::pair{TString{"TxMediator"}, 3},
+    std::pair{TString{"TxAllocator"}, 3},
+    std::pair{TString{"Cms"}, 1},
+    std::pair{TString{"NodeBroker"}, 1},
+    std::pair{TString{"TenantSlotBroker"}, 1},
+    std::pair{TString{"Console"}, 1},
+};
+
+const inline std::map<TString, ui64> GetTablets(ui64 idx) {
+    return {
+        {TString{"FlatHive"}, 72057594037968897},
+        {TString{"FlatBsController"}, 72057594037932033},
+        {TString{"FlatSchemeshard"}, 72057594046678944},
+        {TString{"Cms"}, 72057594037936128},
+        {TString{"NodeBroker"}, 72057594037936129},
+        {TString{"TenantSlotBroker"}, 72057594037936130},
+        {TString{"Console"}, 72057594037936131},
+        {TString{"TxAllocator"}, TDomainsInfo::MakeTxAllocatorIDFixed(1, idx)},
+        {TString{"FlatTxCoordinator"}, TDomainsInfo::MakeTxCoordinatorIDFixed(1, idx)},
+        {TString{"TxMediator"}, TDomainsInfo::MakeTxMediatorIDFixed(1, idx)},
+    };
+}
+
+const inline std::array<std::pair<TString, TString>, 4> DEFAULT_EXECUTORS{
+    std::pair{TString{"IoExecutor"}, TString{"IO"}},
+    std::pair{TString{"SysExecutor"}, TString{"SYSTEM"}},
+    std::pair{TString{"UserExecutor"}, TString{"USER"}},
+    std::pair{TString{"BatchExecutor"}, TString{"BATCH"}},
+};
+
+
+const inline std::array<std::pair<TString, TString>, 3> EXPLICIT_TABLETS{
+    std::pair{TString{"ExplicitCoordinators"}, TString{"FlatTxCoordinator"}},
+    std::pair{TString{"ExplicitAllocators"}, TString{"TxAllocator"}},
+    std::pair{TString{"ExplicitMediators"}, TString{"TxMediator"}},
+};
+
+
+} // namespace NKikimr::NYaml

ydb/library/yaml_config/deprecated/yaml_config_parser.cpp

@@ -6,7 +6,7 @@
 
 #include <library/cpp/json/writer/json.h>
 
-namespace NKikimr::NYaml {
+namespace NKikimr::NYaml::NDeprecated {
 
     template<typename T>
     static bool SetScalarFromYaml(const YAML::Node& yaml, NJson::TJsonValue& json, NJson::EJsonValueType jsonType) {

ydb/library/yaml_config/deprecated/yaml_config_parser.h

@@ -12,7 +12,7 @@
 #include <ydb/core/protos/config.pb.h>
 
 
-namespace NKikimr::NYaml {
+namespace NKikimr::NYaml::NDeprecated {
     NJson::TJsonValue Yaml2Json(const YAML::Node& yaml, bool isRoot);
 
     NKikimrBlobStorage::TConfigRequest BuildInitDistributedStorageCommand(const TString& data);

ydb/library/yaml_config/protos/config.proto

@@ -0,0 +1,84 @@
+syntax = "proto2";
+
+import "ydb/core/config/protos/marker.proto";
+import "ydb/core/protos/blobstorage.proto";
+import "ydb/core/protos/blobstorage_config.proto";
+import "ydb/core/protos/blobstorage_disk.proto";
+import "ydb/core/protos/blobstorage_disk_color.proto";
+import "ydb/core/protos/blobstorage_pdisk_config.proto";
+import "ydb/core/protos/blobstorage_vdisk_config.proto";
+import "ydb/core/protos/bootstrap.proto";
+import "ydb/core/protos/config.proto";
+import "ydb/library/actors/protos/interconnect.proto";
+
+package NKikimrConfig;
+option java_package = "ru.yandex.kikimr.proto";
+
+// message below used to parse part of yaml by following path
+// /blob_storage_config/service_set/groups/*/rings/*/fail_domains/*/vdisk_locations/*
+// later this part is transformed and copied to corresponding config parts
+message TCombinedDiskInfo {
+    option (NMarkers.CombinedType) = true;
+    optional string NodeID = 1; // +allow string // have to copy manually
+    optional uint32 PDiskID = 2 [(NMarkers.CopyTo) = "TVDiskLocation", (NMarkers.CopyTo) = "TPDisk"];
+    optional uint32 VDiskSlotID = 3 [(NMarkers.CopyTo) = "TVDiskLocation"];
+    optional uint64 PDiskGuid = 4 [(NMarkers.CopyTo) = "TVDiskLocation", (NMarkers.CopyTo) = "TPDisk"];
+
+    optional NKikimrBlobStorage.TVDiskID VDiskID = 101 [(NMarkers.CopyTo) = "TVDisk"];
+    optional NKikimrBlobStorage.TVDiskLocation VDiskLocation = 102 [(NMarkers.CopyTo) = "TVDisk"];
+    optional NKikimrBlobStorage.TVDiskKind.EVDiskKind VDiskKind = 103 [(NMarkers.CopyTo) = "TVDisk"];
+    optional bool DoDestroy = 104 [default = false, (NMarkers.CopyTo) = "TVDisk"];
+    optional bool DoWipe = 105 [default = false, (NMarkers.CopyTo) = "TVDisk"];
+    optional NKikimrBlobStorage.EEntityStatus EntityStatus = 106 [(NMarkers.CopyTo) = "TVDisk", (NMarkers.CopyTo) = "TPDisk"];
+    optional string StoragePoolName = 108 [(NMarkers.CopyTo) = "TVDisk"];
+    optional NKikimrBlobStorage.TNodeWardenServiceSet.TVDisk.TDonorMode DonorMode = 109 [(NMarkers.CopyTo) = "TVDisk"];
+    repeated NKikimrBlobStorage.TNodeWardenServiceSet.TVDisk.TDonor Donors = 110 [(NMarkers.CopyTo) = "TVDisk"];
+    optional bool ReadOnly = 111 [(NMarkers.CopyTo) = "TVDisk"];
+
+    optional string Path = 203 [(NMarkers.CopyTo) = "TPDisk"];
+    optional string PDiskCategory = 206; // +allow string // have to copy manually
+    optional NKikimrBlobStorage.TPDiskConfig PDiskConfig = 207 [(NMarkers.CopyTo) = "TPDisk"];
+    optional uint64 InMemoryForTestsBufferBytes = 210 [default = 0, (NMarkers.CopyTo) = "TPDisk"];
+    optional string ExpectedSerial = 211 [(NMarkers.CopyTo) = "TPDisk"];
+    optional NKikimrBlobStorage.TSerialManagementStage.E ManagementStage = 212 [(NMarkers.CopyTo) = "TPDisk"];
+    optional NKikimrBlobStorage.TPDiskSpaceColor.E SpaceColorBorder = 213 [(NMarkers.CopyTo) = "TPDisk"];
+}
+
+message TSystemTablets {
+    option (NMarkers.WithMapType) = true;
+    repeated uint64 DefaultNode = 1;
+    repeated NKikimrConfig.TBootstrap.TTablet FlatHive = 2 [(NMarkers.AsMap) = "Tablets"];
+    repeated NKikimrConfig.TBootstrap.TTablet FlatBsController = 3 [(NMarkers.AsMap) = "Tablets"];
+    repeated NKikimrConfig.TBootstrap.TTablet FlatSchemeshard = 4 [(NMarkers.AsMap) = "Tablets"];
+    repeated NKikimrConfig.TBootstrap.TTablet Cms = 5 [(NMarkers.AsMap) = "Tablets"];
+    repeated NKikimrConfig.TBootstrap.TTablet NodeBroker = 6 [(NMarkers.AsMap) = "Tablets"];
+    repeated NKikimrConfig.TBootstrap.TTablet TenantSlotBroker = 7 [(NMarkers.AsMap) = "Tablets"];
+    repeated NKikimrConfig.TBootstrap.TTablet Console = 8 [(NMarkers.AsMap) = "Tablets"];
+    repeated NKikimrConfig.TBootstrap.TTablet FlatTxCoordinator = 9 [(NMarkers.AsMap) = "Tablets"];
+    repeated NKikimrConfig.TBootstrap.TTablet TxAllocator = 10 [(NMarkers.AsMap) = "Tablets"];
+    repeated NKikimrConfig.TBootstrap.TTablet TxMediator = 11 [(NMarkers.AsMap) = "Tablets"];
+}
+
+message THosts {
+    optional uint32 NodeId = 1;
+    optional uint32 Port = 2;
+    optional uint64 HostConfigId = 3;
+    optional string Host = 4;
+    optional string InterconnectHost = 5;
+    optional NActorsInterconnect.TNodeLocation Location = 6;
+    repeated TStaticNameserviceConfig.TEndpoint Endpoint = 7;
+    optional NActorsInterconnect.TNodeLocation WalleLocation = 8 [deprecated=true];
+    optional string Address = 9;
+    optional string Name = 10;
+}
+
+// these are additional fields used only before transformations
+// after all transformations they shouldnt be used
+// but probably will be saved for debugging purposes
+message TEphemeralInputFields {
+    optional TSystemTablets SystemTablets = 1;
+    optional string StaticErasure = 2;
+    repeated THosts Hosts = 3;
+    repeated NKikimrBlobStorage.TDefineHostConfig HostConfigs = 4;
+    optional uint32 StorageConfigGeneration = 5;
+}

ydb/library/yaml_config/protos/ya.make

@@ -0,0 +1,16 @@
+PROTO_LIBRARY(yaml-config-protos)
+
+SRCS(
+    config.proto
+)
+
+PEERDIR(
+    ydb/core/protos
+    ydb/core/config/protos
+)
+
+CPP_PROTO_PLUGIN0(config_proto_plugin ydb/core/config/tools/protobuf_plugin)
+
+EXCLUDE_TAGS(GO_PROTO JAVA_PROTO)
+
+END()

ydb/library/yaml_config/serialize_deserialize.cpp

@@ -0,0 +1,23 @@
+#include <ydb/library/yaml_config/yaml_config_parser.h>
+
+#include <ydb/core/protos/bootstrap.pb.h>
+
+#include <google/protobuf/descriptor.h>
+
+#include <util/generic/string.h>
+#include <util/string/cast.h>
+
+Y_DECLARE_OUT_SPEC(, NKikimr::NYaml::TCombinedDiskInfoKey, stream, value) {
+    stream << "{" << value.Group << "," << value.Ring << "," << value.FailDomain << "," << value.VDiskLocation << "}";
+}
+
+template <>
+bool TryFromStringImpl<::NKikimrConfig::TBootstrap_ETabletType, char>(const char* value, unsigned long size, NKikimrConfig::TBootstrap_ETabletType& res) {
+    const google::protobuf::EnumDescriptor *descriptor = ::NKikimrConfig::TBootstrap_ETabletType_descriptor();
+    const auto* valueDescriptor = descriptor->FindValueByName(TString(value, size));
+    if (valueDescriptor) {
+        res = static_cast<NKikimrConfig::TBootstrap_ETabletType>(valueDescriptor->number());
+        return true;
+    }
+    return false;
+}

ydb/library/yaml_config/tools/dump/main.cpp

@@ -0,0 +1,18 @@
+#include <ydb/library/yaml_config/deprecated/yaml_config_parser.h>
+#include <ydb/library/yaml_config/tools/util/defaults.h>
+#include <ydb/library/yaml_config/yaml_config_parser.h>
+
+#include <util/generic/string.h>
+#include <util/stream/input.h>
+
+auto main(int argc, char* argv[]) -> int {
+    const TString yaml = Cin.ReadAll();
+    if (argc == 2 && TString("--deprecated") == argv[1]) {
+        NKikimrConfig::TAppConfig config;
+        NKikimr::NYaml::NDeprecated::Parse(yaml, config);
+        Cout << NKikimr::NYaml::NInternal::ProtoToJson(config);
+        return 0;
+    }
+    Cout << NKikimr::NYaml::NInternal::ProtoToJson(NKikimr::NYaml::Parse(yaml));
+    return 0;
+}

ydb/library/yaml_config/tools/dump/ya.make

@@ -0,0 +1,13 @@
+PROGRAM(yaml-to-proto-dump)
+
+SRCS(
+    main.cpp
+)
+
+PEERDIR(
+    ydb/library/yaml_config
+    ydb/library/yaml_config/deprecated
+    ydb/library/yaml_config/tools/util
+)
+
+END()

ydb/library/yaml_config/tools/dump_ds_init/main.cpp

@@ -0,0 +1,18 @@
+#include <ydb/library/yaml_config/deprecated/yaml_config_parser.h>
+#include <ydb/library/yaml_config/yaml_config_parser.h>
+#include <ydb/library/yaml_config/tools/util/defaults.h>
+
+#include <util/generic/string.h>
+#include <util/stream/input.h>
+
+auto main(int argc, char* argv[]) -> int {
+    const TString yaml = Cin.ReadAll();
+    NKikimrBlobStorage::TConfigRequest config;
+    if (argc == 2 && TString("--deprecated") == argv[1]) {
+        config = NKikimr::NYaml::NDeprecated::BuildInitDistributedStorageCommand(yaml);
+    } else {
+        config = NKikimr::NYaml::BuildInitDistributedStorageCommand(yaml);
+    }
+    Cout << NKikimr::NYaml::NInternal::ProtoToJson(config);
+    return 0;
+}

ydb/library/yaml_config/tools/dump_ds_init/ya.make

@@ -0,0 +1,13 @@
+PROGRAM(yaml-to-proto-dump-ds-init)
+
+SRCS(
+    main.cpp
+)
+
+PEERDIR(
+    ydb/library/yaml_config
+    ydb/library/yaml_config/deprecated
+    ydb/library/yaml_config/tools/util
+)
+
+END()

ydb/library/yaml_config/tools/simple_json_diff/__main__.py

@@ -0,0 +1,19 @@
+import json
+import sys
+
+
+def main():
+    with open(sys.argv[1], "r") as fh1:
+        control_data = json.load(fh1)
+
+        with open(sys.argv[2], "r") as fh2:
+            test_data = json.load(fh2)
+
+            a, b = json.dumps(control_data, sort_keys=True), json.dumps(test_data, sort_keys=True)
+
+            return 0 if a == b else 1
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ydb/library/yaml_config/tools/simple_json_diff/ya.make

@@ -0,0 +1,7 @@
+PY3_PROGRAM(simple_json_diff)
+
+PY_SRCS(
+    __main__.py
+)
+
+END()

ydb/library/yaml_config/tools/util/defaults.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <library/cpp/protobuf/json/proto2json.h>
+
+#include <google/protobuf/message.h>
+
+#include <util/generic/string.h>
+
+namespace NKikimr::NYaml::NInternal {
+
+inline TString ProtoToJson(const google::protobuf::Message &resp) {
+    auto config = NProtobufJson::TProto2JsonConfig()
+        .SetFormatOutput(true)
+        .SetEnumMode(NProtobufJson::TProto2JsonConfig::EnumName);
+    return NProtobufJson::Proto2Json(resp, config);
+}
+
+} // namespace NKikimr::NYaml::NInternal

ydb/library/yaml_config/tools/util/ya.make

@@ -0,0 +1,11 @@
+LIBRARY()
+
+SRCS(
+    defaults.h
+)
+
+PEERDIR(
+    library/cpp/protobuf/json
+)
+
+END()

ydb/library/yaml_config/tools/ya.make

@@ -0,0 +1,6 @@
+RECURSE(
+    dump
+    dump_ds_init
+    simple_json_diff
+    util
+)

ydb/library/yaml_config/ut/ya.make

@@ -3,10 +3,7 @@ UNITTEST_FOR(ydb/library/yaml_config)
 SRCS(
     console_dumper_ut.cpp
     yaml_config_ut.cpp
-)
-
-PEERDIR(
-    ydb/library/yaml_config/deprecated
+    yaml_config_parser_ut.cpp
 )
 
 END()