feat: implement per-host rate limiting and statistics

README.md

@@ -423,7 +423,12 @@ Options:
       --default-extension <EXTENSION>
           This is the default file extension that is applied to files without an extension.
 
-          This is useful for files without extensions or with unknown extensions. The extension will be used to determine the file type for processing. Examples: --default-extension md, --default-extension html
+          This is useful for files without extensions or with unknown extensions.
+          The extension will be used to determine the file type for processing.
+
+          Examples:
+            --default-extension md
+            --default-extension html
 
       --dump
           Don't perform any link checking. Instead, dump all the links extracted from inputs that would be checked
@@ -519,10 +524,39 @@ Options:
           You can specify custom headers in the format 'Name: Value'. For example, 'Accept: text/html'.
           This is the same format that other tools like curl or wget use.
           Multiple headers can be specified by using the flag multiple times.
+          The specified headers are used for ALL requests.
+          Use the `hosts` option to configure headers on a per-host basis.
 
       --hidden
           Do not skip hidden directories and files
 
+      --host-concurrency <HOST_CONCURRENCY>
+          Default maximum concurrent requests per host (default: 10)
+
+          This limits the maximum amount of requests that are sent simultaneously
+          to the same host. This helps to prevent overwhelming servers and
+          running into rate-limits. Use the `hosts` option to configure this
+          on a per-host basis.
+
+          Examples:
+            --host-concurrency 2   # Conservative for slow APIs
+            --host-concurrency 20  # Aggressive for fast APIs
+
+      --host-request-interval <HOST_REQUEST_INTERVAL>
+          Minimum interval between requests to the same host (default: 50ms)
+
+          Sets a baseline delay between consecutive requests to prevent
+          overloading servers. The adaptive algorithm may increase this based
+          on server responses (rate limits, errors). Use the `hosts` option
+          to configure this on a per-host basis.
+
+          Examples:
+            --host-request-interval 50ms   # Fast for robust APIs
+            --host-request-interval 1s     # Conservative for rate-limited APIs
+
+      --host-stats
+          Show per-host statistics at the end of the run
+
   -i, --insecure
           Proceed for server connections considered insecure (invalid TLS)
 

fixtures/configs/headers.toml

@@ -4,3 +4,7 @@ X-Bar = "Baz"
 
 # Alternative TOML syntax:
 # header = { X-Foo = "Bar", X-Bar = "Baz" }
+
+
+[hosts."127.0.0.1"]
+headers = { "X-Host-Specific" = "Foo" }

lychee-bin/src/client.rs

@@ -2,7 +2,7 @@ use crate::options::{Config, HeaderMapExt};
 use crate::parse::{parse_duration_secs, parse_remaps};
 use anyhow::{Context, Result};
 use http::{HeaderMap, StatusCode};
-use lychee_lib::{Client, ClientBuilder};
+use lychee_lib::{Client, ClientBuilder, ratelimit::RateLimitConfig};
 use regex::RegexSet;
 use reqwest_cookie_store::CookieStoreMutex;
 use std::sync::Arc;
@@ -55,6 +55,11 @@ pub(crate) fn create(cfg: &Config, cookie_jar: Option<&Arc<CookieStoreMutex>>) -
         .include_fragments(cfg.include_fragments)
         .fallback_extensions(cfg.fallback_extensions.clone())
         .index_files(cfg.index_files.clone())
+        .rate_limit_config(RateLimitConfig::from_options(
+            cfg.host_concurrency,
+            cfg.host_request_interval,
+        ))
+        .hosts(cfg.hosts.clone())
         .build()
         .client()
         .context("Failed to create request client")

lychee-bin/src/commands/check.rs

@@ -1,8 +1,10 @@
 use std::collections::HashSet;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
+use std::sync::Mutex;
 use std::time::Duration;
 
 use futures::StreamExt;
+use lychee_lib::ratelimit::HostPool;
 use reqwest::Url;
 use tokio::sync::mpsc;
 use tokio_stream::wrappers::ReceiverStream;
@@ -24,7 +26,7 @@ use super::CommandParams;
 
 pub(crate) async fn check<S>(
     params: CommandParams<S>,
-) -> Result<(ResponseStats, Arc<Cache>, ExitCode), ErrorKind>
+) -> Result<(ResponseStats, Cache, ExitCode, Arc<HostPool>), ErrorKind>
 where
     S: futures::Stream<Item = Result<Request, RequestError>>,
 {
@@ -41,7 +43,6 @@ where
     } else {
         ResponseStats::default()
     };
-    let cache_ref = params.cache.clone();
 
     let client = params.client;
     let cache = params.cache;
@@ -53,7 +54,7 @@ where
     let accept = params.cfg.accept.into();
 
     // Start receiving requests
-    tokio::spawn(request_channel_task(
+    let handle = tokio::spawn(request_channel_task(
         recv_req,
         send_resp,
         max_concurrency,
@@ -74,8 +75,9 @@ where
         stats,
     ));
 
-    // Wait until all messages are sent
-    send_inputs_loop(params.requests, send_req, &progress).await?;
+    // Wait until all requests are sent
+    send_requests(params.requests, send_req, &progress).await?;
+    let (cache, client) = handle.await?;
 
     // Wait until all responses are received
     let result = show_results_task.await?;
@@ -103,7 +105,8 @@ where
     } else {
         ExitCode::LinkCheckFailure
     };
-    Ok((stats, cache_ref, code))
+
+    Ok((stats, cache, code, client.host_pool()))
 }
 
 async fn suggest_archived_links(
@@ -143,7 +146,7 @@ async fn suggest_archived_links(
 // drops the `send_req` channel on exit
 // required for the receiver task to end, which closes send_resp, which allows
 // the show_results_task to finish
-async fn send_inputs_loop<S>(
+async fn send_requests<S>(
     requests: S,
     send_req: mpsc::Sender<Result<Request, RequestError>>,
     progress: &Progress,
@@ -180,17 +183,17 @@ async fn request_channel_task(
     send_resp: mpsc::Sender<Result<Response, ErrorKind>>,
     max_concurrency: usize,
     client: Client,
-    cache: Arc<Cache>,
+    cache: Cache,
     cache_exclude_status: HashSet<u16>,
     accept: HashSet<u16>,
-) {
+) -> (Cache, Client) {
     StreamExt::for_each_concurrent(
         ReceiverStream::new(recv_req),
         max_concurrency,
         |request: Result<Request, RequestError>| async {
             let response = handle(
                 &client,
-                cache.clone(),
+                &cache,
                 cache_exclude_status.clone(),
                 request,
                 accept.clone(),
@@ -204,6 +207,8 @@ async fn request_channel_task(
         },
     )
     .await;
+
+    (cache, client)
 }
 
 /// Check a URL and return a response.
@@ -235,7 +240,7 @@ async fn check_url(client: &Client, request: Request) -> Response {
 /// a failed response.
 async fn handle(
     client: &Client,
-    cache: Arc<Cache>,
+    cache: &Cache,
     cache_exclude_status: HashSet<u16>,
     request: Result<Request, RequestError>,
     accept: HashSet<u16>,
@@ -247,6 +252,8 @@ async fn handle(
     };
 
     let uri = request.uri.clone();
+
+    // First check the persistent disk-based cache
     if let Some(v) = cache.get(&uri) {
         // Found a cached request
         // Overwrite cache status in case the URI is excluded in the
@@ -260,16 +267,28 @@ async fn handle(
             // code.
             Status::from_cache_status(v.value().status, &accept)
         };
+
+        // Track cache hit in the per-host stats (only for network URIs)
+        if !uri.is_file()
+            && let Err(e) = client.host_pool().record_cache_hit(&uri)
+        {
+            log::debug!("Failed to record cache hit for {uri}: {e}");
+        }
+
         return Ok(Response::new(uri.clone(), status, request.source.into()));
     }
 
-    // Request was not cached; run a normal check
+    // Cache miss - track it and run a normal check (only for network URIs)
+    if !uri.is_file()
+        && let Err(e) = client.host_pool().record_cache_miss(&uri)
+    {
+        log::debug!("Failed to record cache miss for {uri}: {e}");
+    }
+
     let response = check_url(client, request).await;
 
-    // - Never cache filesystem access as it is fast already so caching has no
-    //   benefit.
-    // - Skip caching unsupported URLs as they might be supported in a
-    //   future run.
+    // - Never cache filesystem access as it is fast already so caching has no benefit.
+    // - Skip caching unsupported URLs as they might be supported in a future run.
     // - Skip caching excluded links; they might not be excluded in the next run.
     // - Skip caching links for which the status code has been explicitly excluded from the cache.
     let status = response.status();

lychee-bin/src/commands/mod.rs

@@ -10,7 +10,6 @@ pub(crate) use dump_inputs::dump_inputs;
 use std::fs;
 use std::io::{self, Write};
 use std::path::PathBuf;
-use std::sync::Arc;
 
 use crate::cache::Cache;
 use crate::options::Config;
@@ -20,7 +19,7 @@ use lychee_lib::{Client, Request};
 /// Parameters passed to every command
 pub(crate) struct CommandParams<S: futures::Stream<Item = Result<Request, RequestError>>> {
     pub(crate) client: Client,
-    pub(crate) cache: Arc<Cache>,
+    pub(crate) cache: Cache,
     pub(crate) requests: S,
     pub(crate) cfg: Config,
 }