Add admin cluster config generate command

ydb/core/cms/console/configs_dispatcher.cpp

@@ -181,6 +181,7 @@ class TConfigsDispatcher : public TActorBootstrapped<TConfigsDispatcher> {
     void Handle(TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest::TPtr &ev);
     void Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev);
     void Handle(TEvConsole::TEvGetNodeLabelsRequest::TPtr &ev);
+    void Handle(TEvConsole::TEvFetchStartupConfigRequest::TPtr &ev);
 
     void ReplyMonJson(TActorId mailbox);
 
@@ -199,6 +200,7 @@ class TConfigsDispatcher : public TActorBootstrapped<TConfigsDispatcher> {
             hFuncTraced(TEvConfigsDispatcher::TEvRemoveConfigSubscriptionRequest, Handle);
             // Resolve
             hFunc(TEvConsole::TEvGetNodeLabelsRequest, Handle);
+            hFunc(TEvConsole::TEvFetchStartupConfigRequest, Handle);
         default:
             EnqueueEvent(ev);
             break;
@@ -223,7 +225,7 @@ class TConfigsDispatcher : public TActorBootstrapped<TConfigsDispatcher> {
             IgnoreFunc(TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse);
             // Resolve
             hFunc(TEvConsole::TEvGetNodeLabelsRequest, Handle);
-
+            hFunc(TEvConsole::TEvFetchStartupConfigRequest, Handle);
             // Ignore these console requests until we get rid of persistent subscriptions-related code
             IgnoreFunc(TEvConsole::TEvAddConfigSubscriptionResponse);
             IgnoreFunc(TEvConsole::TEvGetNodeConfigResponse);
@@ -242,6 +244,7 @@ class TConfigsDispatcher : public TActorBootstrapped<TConfigsDispatcher> {
     const std::variant<std::monostate, TDenyList, TAllowList> ItemsServeRules;
     const NKikimrConfig::TAppConfig BaseConfig;
     NKikimrConfig::TAppConfig CurrentConfig;
+    const TString StartupConfigYaml;
     NKikimrConfig::TAppConfig CandidateStartupConfig;
     bool StartupConfigProcessError = false;
     bool StartupConfigProcessDiff = false;
@@ -275,6 +278,7 @@ TConfigsDispatcher::TConfigsDispatcher(const TConfigsDispatcherInitInfo& initInf
         , ItemsServeRules(initInfo.ItemsServeRules)
         , BaseConfig(initInfo.InitialConfig)
         , CurrentConfig(initInfo.InitialConfig)
+        , StartupConfigYaml(initInfo.StartupConfigYaml)
         , CandidateStartupConfig(initInfo.InitialConfig)
         , DebugInfo(initInfo.DebugInfo)
         , RecordedInitialConfiguratorDeps(std::move(initInfo.RecordedInitialConfiguratorDeps))
@@ -1232,6 +1236,15 @@ void TConfigsDispatcher::Handle(TEvConsole::TEvGetNodeLabelsRequest::TPtr &ev) {
     Send(ev->Sender, Response.Release());
 }
 
+void TConfigsDispatcher::Handle(TEvConsole::TEvFetchStartupConfigRequest::TPtr &ev) {
+    auto Response = MakeHolder<TEvConsole::TEvFetchStartupConfigResponse>();
+
+    auto* resp = Response->Record.MutableResponse();
+    resp->set_config(StartupConfigYaml);
+
+    Send(ev->Sender, Response.Release());
+}
+
 IActor *CreateConfigsDispatcher(const TConfigsDispatcherInitInfo& initInfo) {
     return new TConfigsDispatcher(initInfo);
 }

ydb/core/cms/console/console.h

@@ -56,7 +56,7 @@ namespace TEvConsole {
         EvGetAllMetadataRequest,
         EvGetNodeLabelsRequest,
         EvIsYamlReadOnlyRequest,
-
+        EvFetchStartupConfigRequest,
         // responses
         EvCreateTenantResponse = EvCreateTenantRequest + 1024,
         EvAlterTenantResponse,
@@ -102,6 +102,7 @@ namespace TEvConsole {
         EvGenericError,
 
         EvIsYamlReadOnlyResponse,
+        EvFetchStartupConfigResponse,
 
         EvEnd
     };
@@ -225,6 +226,10 @@ namespace TEvConsole {
 
     struct TEvResolveAllConfigResponse : public TEventShortDebugPB<TEvResolveAllConfigResponse, NKikimrConsole::TResolveAllConfigResponse, EvResolveAllConfigResponse> {};
 
+    struct TEvFetchStartupConfigRequest : public TEventShortDebugPB<TEvFetchStartupConfigRequest, NKikimrConsole::TFetchStartupConfigRequest, EvFetchStartupConfigRequest> {};
+
+    struct TEvFetchStartupConfigResponse : public TEventShortDebugPB<TEvFetchStartupConfigResponse, NKikimrConsole::TFetchStartupConfigResponse, EvFetchStartupConfigResponse> {};
+
     struct TEvUnauthorized : public TEventShortDebugPB<TEvUnauthorized, NKikimrConsole::TUnauthorized, EvUnauthorized> {};
 
     struct TEvDisabled : public TEventShortDebugPB<TEvDisabled, NKikimrConsole::TDisabled, EvDisabled> {};

ydb/core/cms/console/console_configs_manager.cpp

@@ -861,6 +861,11 @@ void TConfigsManager::Handle(TEvConsole::TEvGetNodeLabelsRequest::TPtr &ev, cons
     }
 }
 
+void TConfigsManager::Handle(TEvConsole::TEvFetchStartupConfigRequest::TPtr &ev, const TActorContext &ctx)
+{
+    ctx.Send(ev->Forward(MakeConfigsDispatcherID(SelfId().NodeId())));
+}
+
 void TConfigsManager::Handle(TEvConsole::TEvGetAllMetadataRequest::TPtr &ev, const TActorContext &ctx)
 {
     TxProcessor->ProcessTx(CreateTxGetYamlMetadata(ev), ctx);

ydb/core/cms/console/console_configs_manager.h

@@ -193,6 +193,7 @@ class TConfigsManager : public TActorBootstrapped<TConfigsManager> {
     void Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev, const TActorContext &ctx);
     void Handle(TEvConsole::TEvReplaceYamlConfigRequest::TPtr & ev, const TActorContext & ctx);
     void Handle(TEvConsole::TEvSetYamlConfigRequest::TPtr & ev, const TActorContext & ctx);
+    void Handle(TEvConsole::TEvFetchStartupConfigRequest::TPtr & ev, const TActorContext & ctx);
     void HandleUnauthorized(TEvConsole::TEvReplaceYamlConfigRequest::TPtr & ev, const TActorContext & ctx);
     void HandleUnauthorized(TEvConsole::TEvSetYamlConfigRequest::TPtr & ev, const TActorContext & ctx);
     void Handle(TEvConsole::TEvDropConfigRequest::TPtr & ev, const TActorContext & ctx);
@@ -248,6 +249,7 @@ class TConfigsManager : public TActorBootstrapped<TConfigsManager> {
             HFunc(TEvConsole::TEvGetAllConfigsRequest, HandleWithRights);
             HFunc(TEvConsole::TEvGetNodeLabelsRequest, HandleWithRights);
             HFunc(TEvConsole::TEvGetAllMetadataRequest, HandleWithRights);
+            HFunc(TEvConsole::TEvFetchStartupConfigRequest, HandleWithRights);
             HFunc(TEvConsole::TEvAddVolatileConfigRequest, HandleWithRights);
             HFunc(TEvConsole::TEvRemoveVolatileConfigRequest, HandleWithRights);
             FFunc(TEvConsole::EvGetConfigItemsRequest, ForwardToConfigsProvider);

ydb/core/cms/console/console_impl.h

@@ -114,6 +114,7 @@ class TConsole : public TActor<TConsole>
             FFunc(TEvConsole::EvDropConfigRequest, ForwardToConfigsManager);
             FFunc(TEvConsole::EvResolveConfigRequest, ForwardToConfigsManager);
             FFunc(TEvConsole::EvResolveAllConfigRequest, ForwardToConfigsManager);
+            FFunc(TEvConsole::EvFetchStartupConfigRequest, ForwardToConfigsManager);
             FFunc(TEvConsole::EvGetConfigSubscriptionRequest, ForwardToConfigsManager);
             FFunc(TEvConsole::EvGetNodeConfigItemsRequest, ForwardToConfigsManager);
             FFunc(TEvConsole::EvGetNodeConfigRequest, ForwardToConfigsManager);

ydb/core/config/init/init.cpp

@@ -591,6 +591,7 @@ void LoadMainYamlConfig(TConfigRefs refs, const TString& mainYamlConfigFile, NKi
     IProtoConfigFileProvider& protoConfigFileProvider = refs.ProtoConfigFileProvider;
 
     const TString mainYamlConfigString = protoConfigFileProvider.GetProtoFromFile(mainYamlConfigFile, errorCollector);
+    appConfig.SetStartupConfigYaml(mainYamlConfigString);
 
     if (appConfig.GetSelfManagementConfig().GetEnabled()) {
         // fill in InitialConfigYaml only when self-management through distconf is enabled

ydb/core/config/init/init.h

@@ -228,6 +228,7 @@ struct TDebugInfo {
 
 struct TConfigsDispatcherInitInfo {
     NKikimrConfig::TAppConfig InitialConfig;
+    TString StartupConfigYaml;
     TMap<TString, TString> Labels;
     std::variant<std::monostate, TDenyList, TAllowList> ItemsServeRules;
     std::optional<TDebugInfo> DebugInfo;

ydb/core/config/init/init_impl.h

@@ -1396,6 +1396,7 @@ class TInitialConfiguratorImpl
         servicesMask = ServicesMask;
         clusterName = ClusterName;
         configsDispatcherInitInfo.InitialConfig = appConfig;
+        configsDispatcherInitInfo.StartupConfigYaml = appConfig.GetStartupConfigYaml();
         configsDispatcherInitInfo.ItemsServeRules = std::monostate{},
         configsDispatcherInitInfo.Labels = Labels;
         configsDispatcherInitInfo.DebugInfo = TDebugInfo {

ydb/core/grpc_services/rpc_dynamic_config.cpp

@@ -43,6 +43,9 @@ using TEvResolveConfigRequest = TGrpcRequestOperationCall<DynamicConfig::Resolve
 using TEvResolveAllConfigRequest = TGrpcRequestOperationCall<DynamicConfig::ResolveAllConfigRequest,
     DynamicConfig::ResolveAllConfigResponse>;
 
+using TEvFetchStartupConfigRequest = TGrpcRequestOperationCall<DynamicConfig::FetchStartupConfigRequest,
+    DynamicConfig::FetchStartupConfigResponse>;
+
 template <typename TRequest, typename TConsoleRequest, typename TConsoleResponse>
 class TDynamicConfigRPC : public TRpcOperationRequestActor<TDynamicConfigRPC<TRequest, TConsoleRequest, TConsoleResponse>, TRequest> {
     using TThis = TDynamicConfigRPC<TRequest, TConsoleRequest, TConsoleResponse>;
@@ -121,6 +124,10 @@ class TDynamicConfigRPC : public TRpcOperationRequestActor<TDynamicConfigRPC<TRe
         return TBase::ReplyWithResult(Ydb::StatusIds::SUCCESS, ev->Get()->Record.GetResponse(), TActivationContext::AsActorContext());
     }
 
+    void Handle(TEvConsole::TEvFetchStartupConfigResponse::TPtr& ev) {
+        return TBase::ReplyWithResult(Ydb::StatusIds::SUCCESS, ev->Get()->Record.GetResponse(), TActivationContext::AsActorContext());
+    }
+
     void Handle(TEvConsole::TEvUnauthorized::TPtr&) {
         ::google::protobuf::RepeatedPtrField< ::Ydb::Issue::IssueMessage> issues;
         auto issue = issues.Add();
@@ -283,4 +290,10 @@ void DoResolveAllConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacility
                     TEvConsole::TEvResolveAllConfigResponse>(p.release()));
 }
 
+void DoFetchStartupConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider &) {
+    TActivationContext::AsActorContext().Register(
+        new TDynamicConfigRPC<TEvFetchStartupConfigRequest,
+                    TEvConsole::TEvFetchStartupConfigRequest,
+                    TEvConsole::TEvFetchStartupConfigResponse>(p.release()));
+}
 } // namespace NKikimr::NGRpcService

ydb/core/grpc_services/service_dynamic_config.h

@@ -28,5 +28,7 @@ void DoResolveConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityPro
 
 void DoResolveAllConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&);
 
+void DoFetchStartupConfigRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&);
+
 }
 }

ydb/core/protos/config.proto

@@ -2238,6 +2238,7 @@ message TAppConfig {
     optional string ConfigStorePath = 104;
     optional NKikimrBlobStorage.TYamlConfig StoredConfigYaml = 105;
     optional bool ConfigLoadedFromStore = 106;
+    optional string StartupConfigYaml = 107;
 }
 
 message TYdbVersion {

ydb/core/protos/console_config.proto

@@ -646,6 +646,17 @@ message TToggleConfigValidatorResponse {
     optional TStatus Status = 1;
 }
 
+message TFetchStartupConfigRequest {
+    optional Ydb.DynamicConfig.FetchStartupConfigRequest Request = 1;
+    optional bytes UserToken = 2;
+    optional string PeerName = 3;
+    optional string IngressDatabase = 4;
+}
+
+message TFetchStartupConfigResponse {
+    optional Ydb.DynamicConfig.FetchStartupConfigResult Response = 1;
+}
+
 enum EValidationLevel {
     // Validation is disabled.
     VALIDATE_NONE = 0;

ydb/library/yaml_config/public/yaml_config.cpp

@@ -801,6 +801,11 @@ bool IsDatabaseConfig(const TString& config) {
     return IsConfigKindEquals(config, "DatabaseConfig");
 }
 
+bool IsStaticConfig(const TString& config) {
+    auto doc = NFyaml::TDocument::Parse(config);
+    return !doc.Root().Map().Has("metadata");
+}
+
 TString StripMetadata(const TString& config) {
     auto doc = NFyaml::TDocument::Parse(config);
 

ydb/library/yaml_config/public/yaml_config.h

@@ -262,6 +262,11 @@ bool IsStorageConfig(const TString& config);
  */
 bool IsDatabaseConfig(const TString& config);
 
+/**
+ * Checks whether string is static config or not
+ */
+bool IsStaticConfig(const TString& config);
+
 /**
  * Strips metadata from config
  */

ydb/public/api/grpc/draft/ydb_dynamic_config_v1.proto

@@ -46,4 +46,7 @@ service DynamicConfigService {
 
     // Resolve config for all possible labels combinations.
     rpc ResolveAllConfig(DynamicConfig.ResolveAllConfigRequest) returns (DynamicConfig.ResolveAllConfigResponse);
+
+    // Generate dynamic config based on cluster's static config.
+    rpc FetchStartupConfig(DynamicConfig.FetchStartupConfigRequest) returns (DynamicConfig.FetchStartupConfigResponse);
 }

ydb/public/api/protos/draft/ydb_dynamic_config.proto

@@ -228,3 +228,17 @@ message ResolveAllConfigResult {
     // Verbose resolved configs
     repeated ResolvedConfig configs = 2;
 }
+
+message FetchStartupConfigRequest {
+    Ydb.Operations.OperationParams operation_params = 1;
+}
+
+message FetchStartupConfigResponse {
+    // Result of request will be inside operation.
+    Ydb.Operations.Operation operation = 1;
+}
+
+message FetchStartupConfigResult {
+    // YAML document with generated dynamic config
+    string config = 1;
+}

ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp

@@ -22,6 +22,21 @@ TString WrapYaml(const TString& yaml) {
     return out.Str();
 }
 
+TString WrapStaticConfig(const TString& yaml) {
+    auto newDoc = NFyaml::TDocument::Parse("---\nmetadata: {}\nconfig: {}\n");
+    auto inputDoc = NFyaml::TDocument::Parse(yaml);
+
+    auto configNode = inputDoc.Root().Copy(newDoc);
+    newDoc.Root().Map().pair_at("config").SetValue(configNode.Ref());
+
+    auto metadataNode = newDoc.Root().Map().pair_at("metadata").Value().Map();
+    metadataNode.Append(newDoc.Buildf("kind"), newDoc.Buildf("MainConfig"));
+    metadataNode.Append(newDoc.Buildf("cluster"), newDoc.Buildf("\"\""));
+    metadataNode.Append(newDoc.Buildf("version"), newDoc.Buildf("0"));
+
+    return TString(newDoc.EmitToCharArray().get());
+}
+
 TCommandConfig::TCommandConfig(
         TCommandFlagsOverrides commandFlagsOverrides,
         bool allowEmptyDatabase)
@@ -31,6 +46,7 @@ TCommandConfig::TCommandConfig(
     AddCommand(std::make_unique<TCommandConfigFetch>(allowEmptyDatabase));
     AddCommand(std::make_unique<TCommandConfigReplace>(allowEmptyDatabase));
     AddCommand(std::make_unique<TCommandConfigResolve>());
+    AddCommand(std::make_unique<TCommandGenerateDynamicConfig>(allowEmptyDatabase));
 }
 
 TCommandConfig::TCommandConfig(bool allowEmptyDatabase)
@@ -688,4 +704,31 @@ int TCommandConfigVolatileFetch::Run(TConfig& config) {
     return EXIT_SUCCESS;
 }
 
+TCommandGenerateDynamicConfig::TCommandGenerateDynamicConfig(bool allowEmptyDatabase)
+    : TYdbReadOnlyCommand("generate", {}, "Generate dynamic config from startup static config")
+    , AllowEmptyDatabase(allowEmptyDatabase)
+{
+}
+
+void TCommandGenerateDynamicConfig::Config(TConfig& config) {
+    TYdbCommand::Config(config);
+    config.SetFreeArgsNum(0);
+    config.AllowEmptyDatabase = AllowEmptyDatabase;
+}
+
+int TCommandGenerateDynamicConfig::Run(TConfig& config) {
+    auto driver = std::make_unique<NYdb::TDriver>(CreateDriver(config));
+    auto client = NYdb::NDynamicConfig::TDynamicConfigClient(*driver);
+
+    auto result = client.FetchStartupConfig().GetValueSync();
+    NStatusHelpers::ThrowOnErrorOrPrintIssues(result);
+    if (NYamlConfig::IsStaticConfig(TString{result.GetConfig()})) {
+        Cout << WrapStaticConfig(TString{result.GetConfig()});
+    } else {
+        Cout << "Startup config is already dynamic" << Endl;
+    }
+
+    return EXIT_SUCCESS;
+}
+
 } // namespace NYdb::NConsoleClient::NDynamicConfig

ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h

@@ -124,4 +124,13 @@ class TCommandConfigVolatileFetch : public TYdbCommand {
     bool StripMetadata = false;
 };
 
+class TCommandGenerateDynamicConfig : public TYdbReadOnlyCommand {
+public:
+    TCommandGenerateDynamicConfig(bool allowEmptyDatabase);
+    void Config(TConfig&) override;
+    int Run(TConfig&) override;
+private:
+    bool AllowEmptyDatabase = false;
+};
+
 } // namespace NYdb::NConsoleClient::NDynamicConfig

ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/draft/ydb_dynamic_config.h

@@ -160,6 +160,21 @@ struct TVerboseResolveConfigResult : public TStatus {
 
 using TAsyncVerboseResolveConfigResult = NThreading::TFuture<TVerboseResolveConfigResult>;
 
+struct TFetchStartupConfigResult : public TStatus {
+    TFetchStartupConfigResult(TStatus&& status, std::string&& config)
+        : TStatus(std::move(status))
+        , Config_(std::move(config))
+    {}
+
+    const std::string& GetConfig() const {
+        return Config_;
+    }
+
+private:
+    std::string Config_;
+};
+
+using TAsyncFetchStartupConfigResult = NThreading::TFuture<TFetchStartupConfigResult>;
 
 struct TDynamicConfigClientSettings : public TCommonClientSettingsBase<TDynamicConfigClientSettings> {
     using TSelf = TDynamicConfigClientSettings;
@@ -240,6 +255,9 @@ class TDynamicConfigClient {
         const std::map<uint64_t, std::string>& volatileConfigs,
         const TClusterConfigSettings& settings = {});
 
+    // Fetch startup config
+    TAsyncFetchStartupConfigResult FetchStartupConfig(const TClusterConfigSettings& settings = {});
+
 private:
     std::shared_ptr<TImpl> Impl_;
 };