refactor(rate limiting): allow custom rate limits

src/constants.rs

@@ -1,5 +1,5 @@
 use lazy_static::lazy_static;
-use std::sync::RwLock;
+use std::{collections::HashMap, sync::RwLock};
 
 pub const MAX_NAME_LENGTH: usize = 32;
 pub const MAX_URI_LENGTH: usize = 200;
@@ -9,23 +9,21 @@ pub const MAX_CREATOR_LEN: usize = 32 + 1 + 1;
 pub const METAPLEX_PROGRAM_ID: &'static str = "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s";
 pub const CANDY_MACHINE_PROGRAM_ID: &'static str = "cndyAnrLdpjq1Ssp1z8xxDsB8dxe7u4HL5Nxi2K5WXZ";
 
-pub const DEFAULT_RPC_DELAY_MS: u64 = 300;
-
 pub const PUBLIC_RPC_URLS: &'static [&'static str] = &[
     "https://api.devnet.solana.com",
     "https://api.testnet.solana.com",
     "https://api.mainnet-beta.solana.com",
     "https://solana-api.projectserum.com",
-    "https://ssc-dao.genesysgo.net",
 ];
 
-pub const MAX_REQUESTS: u64 = 40;
-pub const TIME_PER_MAX_REQUESTS_NS: u64 = 10_000_000_000;
-pub const TIME_BUFFER_NS: u32 = 50_000_000;
-
-// Delay in milliseconds between RPC requests
-pub const RATE_LIMIT: u64 = 500;
+pub const DEFAULT_RPC_DELAY_MS: u32 = 200;
 
 lazy_static! {
     pub static ref USE_RATE_LIMIT: RwLock<bool> = RwLock::new(false);
+    pub static ref RPC_DELAY_NS: RwLock<u32> = RwLock::new(DEFAULT_RPC_DELAY_MS * 1_000_000);
+    pub static ref RATE_LIMIT_DELAYS: HashMap<&'static str, u32> =
+        [("https://ssc-dao.genesysgo.net", 25),]
+            .iter()
+            .copied()
+            .collect();
 }

src/limiter.rs

@@ -9,10 +9,7 @@ pub fn create_rate_limiter() -> Handle {
     let mut limiter = ratelimit::Builder::new()
         .capacity(num_cpus as u32)
         .quantum(1)
-        .interval(Duration::new(
-            0,
-            (TIME_PER_MAX_REQUESTS_NS / MAX_REQUESTS) as u32 + TIME_BUFFER_NS,
-        ))
+        .interval(Duration::new(0, *RPC_DELAY_NS.read().unwrap()))
         .build();
 
     let handle = limiter.make_handle();

src/main.rs

@@ -39,7 +39,7 @@ fn main() -> Result<()> {
             (config.json_rpc_url, config.commitment)
         } else {
             info!(
-            "Could not find a valid Solana-CLI config file. Defaulting to https://psytrbhymqlkfrhudd.dev.genesysgo.net:8899/ devenet node."
+            "Could not find a valid Solana-CLI config file. Defaulting to https://psytrbhymqlkfrhudd.dev.genesysgo.net:8899/ devnet node."
         );
             (
                 String::from("https://psytrbhymqlkfrhudd.dev.genesysgo.net:8899/"),
@@ -48,13 +48,23 @@ fn main() -> Result<()> {
         }
     };
 
+    // Remove trailing slash
+    let rpc = if rpc.ends_with('/') {
+        rpc.trim_end_matches('/').to_string()
+    } else {
+        rpc
+    };
+
     // Set rate limiting if the user specified a public RPC.
     if PUBLIC_RPC_URLS.contains(&rpc.as_str()) {
         warn!(
             "Using a public RPC URL is not recommended for heavy tasks as you will be rate-limited and suffer a performance hit.
         Please use a private RPC endpoint for best performance results."
         );
         *USE_RATE_LIMIT.write().unwrap() = true;
+    } else if RATE_LIMIT_DELAYS.contains_key(&rpc.as_str()) {
+        *USE_RATE_LIMIT.write().unwrap() = true;
+        *RPC_DELAY_NS.write().unwrap() = RATE_LIMIT_DELAYS[&rpc.as_str()];
     }
 
     let commitment = CommitmentConfig::from_str(&commitment)?;

src/snapshot.rs

@@ -97,8 +97,14 @@ pub fn snapshot_mints(
     println!("Getting metadata and writing to file...");
     let mut mint_accounts: Vec<String> = Vec::new();
 
-    for (_, account) in accounts {
-        let metadata: Metadata = try_from_slice_unchecked(&account.data)?;
+    for (pubkey, account) in accounts {
+        let metadata: Metadata = match try_from_slice_unchecked(&account.data) {
+            Ok(metadata) => metadata,
+            Err(_) => {
+                error!("Failed to parse metadata for account {}", pubkey);
+                continue;
+            }
+        };
 
         if first_creator_is_verified(&metadata.data.creators) {
             mint_accounts.push(metadata.mint.to_string());