feat: Update neard init to specify config file type to download (#12072)

We change the `--download-config` parameter to take a string.
- `neard init --download-config validator` downloads the `config.json`
file for validators (tracking single shard, loading memtries).
- `neard init --download-config rpc` downloads the `config.json` file
for non-validator RPC nodes (tracking all shards, NOT loading memtries).
- `neard init --download-config archival` downloads the `config.json`
file for non-validator archival nodes (tracking all shards, NOT loading
memtries, archive field set to true).
- `neard init --download-config` (with no value) defaults to `validator`
config (for backwards compatibility).

We map the flag to a specific URL containing the config type. 
The new config files per config type can be found at
https://github.com/tayfunelmas/near-configs.
The new config files are already uploaded to AWS S3 with the new URLs.

NOTE: For indexer and chainsync-loadtest params, we use `rpc` config
type.

Next: We will update the Near validator documentation to use the new
flag value.

Tracking issue: #12070

chain/indexer/Cargo.toml

@@ -24,6 +24,7 @@ tracing.workspace = true
 nearcore.workspace = true
 near-client.workspace = true
 near-chain-configs.workspace = true
+near-config-utils.workspace = true
 near-dyn-configs.workspace = true
 near-crypto.workspace = true
 near-indexer-primitives.workspace = true

chain/indexer/src/lib.rs

@@ -1,6 +1,7 @@
 #![doc = include_str!("../README.md")]
 
 use anyhow::Context;
+use near_config_utils::DownloadConfigType;
 use tokio::sync::mpsc;
 
 use near_chain_configs::GenesisValidationMode;
@@ -43,7 +44,7 @@ pub struct InitConfigArgs {
     /// Specify a custom download URL for the records file.
     pub download_records_url: Option<String>,
     /// Download the verified NEAR config file automatically.
-    pub download_config: bool,
+    pub download_config: Option<DownloadConfigType>,
     /// Specify a custom download URL for the config file.
     pub download_config_url: Option<String>,
     /// Specify the boot nodes to bootstrap the network

nearcore/src/config.rs

@@ -28,7 +28,7 @@ use near_chain_configs::{
     NUM_BLOCK_PRODUCER_SEATS, PROTOCOL_REWARD_RATE, PROTOCOL_UPGRADE_STAKE_THRESHOLD,
     TRANSACTION_VALIDITY_PERIOD,
 };
-use near_config_utils::{ValidationError, ValidationErrors};
+use near_config_utils::{DownloadConfigType, ValidationError, ValidationErrors};
 use near_crypto::{InMemorySigner, KeyFile, KeyType, PublicKey};
 use near_epoch_manager::EpochManagerHandle;
 #[cfg(feature = "json_rpc")]
@@ -811,7 +811,7 @@ pub fn init_configs(
     should_download_genesis: bool,
     download_genesis_url: Option<&str>,
     download_records_url: Option<&str>,
-    should_download_config: bool,
+    download_config_type: Option<DownloadConfigType>,
     download_config_url: Option<&str>,
     boot_nodes: Option<&str>,
     max_gas_burnt_view: Option<Gas>,
@@ -853,8 +853,8 @@ pub fn init_configs(
         download_config(url, &dir.join(CONFIG_FILENAME))
             .context(format!("Failed to download the config file from {}", url))?;
         config = Config::from_file(&dir.join(CONFIG_FILENAME))?;
-    } else if should_download_config {
-        let url = get_config_url(&chain_id);
+    } else if let Some(config_type) = download_config_type {
+        let url = get_config_url(&chain_id, config_type);
         download_config(&url, &dir.join(CONFIG_FILENAME))
             .context(format!("Failed to download the config file from {}", url))?;
         config = Config::from_file(&dir.join(CONFIG_FILENAME))?;
@@ -1328,10 +1328,10 @@ pub fn get_records_url(chain_id: &str) -> String {
     )
 }
 
-pub fn get_config_url(chain_id: &str) -> String {
+pub fn get_config_url(chain_id: &str, config_type: DownloadConfigType) -> String {
     format!(
-        "https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/{}/config.json",
-        chain_id,
+        "https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/{}/{}/config.json",
+        chain_id, config_type.to_string()
     )
 }
 
@@ -1544,7 +1544,7 @@ mod tests {
             false,
             None,
             None,
-            false,
+            None,
             None,
             None,
             None,
@@ -1602,7 +1602,7 @@ mod tests {
             false,
             None,
             None,
-            false,
+            None,
             None,
             None,
             None,
@@ -1635,7 +1635,7 @@ mod tests {
             false,
             None,
             None,
-            false,
+            None,
             None,
             None,
             None,

neard/src/cli.rs

@@ -4,6 +4,7 @@ use near_amend_genesis::AmendGenesisCommand;
 use near_chain_configs::GenesisValidationMode;
 use near_client::ConfigUpdater;
 use near_cold_store_tool::ColdStoreCommand;
+use near_config_utils::DownloadConfigType;
 use near_database_tool::commands::DatabaseCommand;
 use near_dyn_configs::{UpdateableConfigLoader, UpdateableConfigLoaderError, UpdateableConfigs};
 use near_flat_storage::commands::FlatStorageCommand;
@@ -32,6 +33,7 @@ use std::fs::File;
 use std::io::BufReader;
 use std::net::SocketAddr;
 use std::path::{Path, PathBuf};
+use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::broadcast;
 use tokio::sync::broadcast::Receiver;
@@ -259,8 +261,10 @@ pub(super) struct InitCmd {
     #[clap(long)]
     download_genesis: bool,
     /// Download the verified NEAR config file automatically.
-    #[clap(long)]
-    download_config: bool,
+    /// Can be one of "validator", "rpc", and "archival".
+    /// If flag is present with no value, defaults to "validator".
+    #[clap(long, default_missing_value = "validator", num_args(0..=1))]
+    download_config: Option<String>,
     /// Makes block production fast (TESTING ONLY).
     #[clap(long)]
     fast: bool,
@@ -347,6 +351,12 @@ impl InitCmd {
             check_release_build(chain)
         }
 
+        let download_config_type = if let Some(config_type) = self.download_config.as_deref() {
+            Some(DownloadConfigType::from_str(config_type)?)
+        } else {
+            None
+        };
+
         nearcore::init_configs(
             home_dir,
             self.chain_id,
@@ -358,7 +368,7 @@ impl InitCmd {
             self.download_genesis,
             self.download_genesis_url.as_deref(),
             self.download_records_url.as_deref(),
-            self.download_config,
+            download_config_type,
             self.download_config_url.as_deref(),
             self.boot_nodes.as_deref(),
             self.max_gas_burnt_view,

runtime/runtime-params-estimator/src/main.rs

@@ -177,7 +177,7 @@ fn run_estimation(cli_args: CliArgs) -> anyhow::Result<Option<CostTable>> {
             false,
             None,
             None,
-            false,
+            None,
             None,
             None,
             None,

tools/chainsync-loadtest/Cargo.toml

@@ -31,6 +31,7 @@ tokio.workspace = true
 
 near-async.workspace = true
 near-chain-configs.workspace = true
+near-config-utils.workspace = true
 near-crypto.workspace = true
 near-primitives.workspace = true
 near-store.workspace = true

tools/chainsync-loadtest/src/main.rs

@@ -59,7 +59,7 @@ pub fn start_with_config(config: NearConfig, qps_limit: u32) -> anyhow::Result<A
 fn download_configs(chain_id: &str, dir: &std::path::Path) -> anyhow::Result<NearConfig> {
     // Always fetch the config.
     std::fs::create_dir_all(dir)?;
-    let url = config::get_config_url(chain_id);
+    let url = config::get_config_url(chain_id, near_config_utils::DownloadConfigType::RPC);
     let config_path = &dir.join(config::CONFIG_FILENAME);
     config::download_config(&url, config_path)?;
     let config = config::Config::from_file(config_path)?;

tools/indexer/example/Cargo.toml

@@ -20,5 +20,6 @@ serde_json.workspace = true
 tokio.workspace = true
 tracing.workspace = true
 
+near-config-utils.workspace = true
 near-indexer.workspace = true
 near-o11y.workspace = true

tools/indexer/example/src/configs.rs

@@ -76,7 +76,11 @@ impl From<InitConfigArgs> for near_indexer::InitConfigArgs {
             download_genesis: config_args.download_genesis,
             download_genesis_url: config_args.download_genesis_url,
             download_records_url: config_args.download_records_url,
-            download_config: config_args.download_config,
+            download_config: if config_args.download_config {
+                Some(near_config_utils::DownloadConfigType::RPC)
+            } else {
+                None
+            },
             download_config_url: config_args.download_config_url,
             boot_nodes: config_args.boot_nodes,
             max_gas_burnt_view: config_args.max_gas_burnt_view,

utils/config/src/lib.rs

@@ -1,4 +1,4 @@
-use std::io::Read;
+use std::{io::Read, str::FromStr};
 
 use json_comments::StripComments;
 
@@ -134,3 +134,40 @@ impl ValidationErrors {
         }
     }
 }
+
+/// Type of the configuration to download.
+/// Currently used for downloading the `config.json` file for different node types.
+#[derive(Debug, Clone)]
+pub enum DownloadConfigType {
+    /// Validator node configuration.
+    Validator,
+    /// Non-validator RPC node configuration.
+    RPC,
+    /// Non-validator archival node configuration.
+    Archival,
+}
+
+impl ToString for DownloadConfigType {
+    fn to_string(&self) -> String {
+        match self {
+            DownloadConfigType::Validator => "validator".to_string(),
+            DownloadConfigType::RPC => "rpc".to_string(),
+            DownloadConfigType::Archival => "archival".to_string(),
+        }
+    }
+}
+
+impl FromStr for DownloadConfigType {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.trim().to_lowercase().as_str() {
+            "validator" => Ok(DownloadConfigType::Validator),
+            "rpc" => Ok(DownloadConfigType::RPC),
+            "archival" => Ok(DownloadConfigType::Archival),
+            _ => anyhow::bail!(
+                "Flag download_config must be one of the following: validator, rpc, archival"
+            ),
+        }
+    }
+}