feat: Add support for ranges in the --accept option / config field (#…

…1167)

Adds support for accept ranges discussed in #1157. This allows the user to specify custom HTTP status codes accepted during checking and thus will report as valid (not broken). The accept option only supports specifying status codes as a comma-separated list. With this PR, the option will accept a list of status code ranges formatted like this:

```toml
accept = ["100..=103", "200..=299", "403"]
```

These combinations will be supported: `..<end>`, ` ..=<end>`, `<start>..<end>` and `<start>..=<end>`.
The behavior is copied from the Rust Range like concepts:

```
    ..<end>, includes 0 to <end> (exclusive)
    ..=<end>, includes 0 to <end> (inclusive)
    <start>..<end>, includes <start> to <end> (exclusive)
    <start>..=<end>, includes <start> to <end> (inclusive)
```


- Foundation and enhancements for accept ranges, including support for comma-separated strings and integration into the CLI.
- Implementations and updates for AcceptSelector, including Default, Display, and serde defaults.
- Address and fix various errors: clippy, cargo fmt, and tests.
- Add more tests, address edge cases, and enhance error messaging, especially for TOML config parsing.
- Update dependencies.

README.md

@@ -20,9 +20,30 @@ Available as a command-line utility, a library and a [GitHub Action](https://git
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 ## Table of Contents
 
+- [Table of Contents](#table-of-contents)
 - [Installation](#installation)
+  - [Arch Linux](#arch-linux)
+  - [macOS](#macos)
+  - [Docker](#docker)
+  - [NixOS](#nixos)
+  - [FreeBSD](#freebsd)
+  - [Scoop](#scoop)
+  - [Termux](#termux)
+  - [Pre-built binaries](#pre-built-binaries)
+  - [Cargo](#cargo)
+    - [Build dependencies](#build-dependencies)
+    - [Compile and install lychee](#compile-and-install-lychee)
+    - [Feature flags](#feature-flags)
 - [Features](#features)
 - [Commandline usage](#commandline-usage)
+  - [Docker Usage](#docker-usage)
+    - [Linux/macOS shell command](#linuxmacos-shell-command)
+    - [Windows PowerShell command](#windows-powershell-command)
+  - [GitHub Token](#github-token)
+  - [Commandline Parameters](#commandline-parameters)
+  - [Exit codes](#exit-codes)
+  - [Ignoring links](#ignoring-links)
+  - [Caching](#caching)
 - [Library usage](#library-usage)
 - [GitHub Action Usage](#github-action-usage)
 - [Contributing to lychee](#contributing-to-lychee)
@@ -384,7 +405,22 @@ Options:
           Custom request header
 
   -a, --accept <ACCEPT>
-          Comma-separated list of accepted status codes for valid links
+          A List of accepted status codes for valid links
+          
+          The following accept range syntax is supported: [start]..[=]end|code. Some valid
+          examples are:
+          
+          - 200..=204
+          - 200..204
+          - ..=204
+          - ..204
+          - 200
+          
+          Use "lychee --accept '200..=204, 429, 500' <inputs>..." to provide a comma-
+          separated list of accepted status codes. This example will accept 200, 201,
+          202, 203, 204, 429, and 500 as valid status codes.
+          
+          [default: 100..=103,200..=299,403..=403]
 
       --include-fragments
           Enable the checking of fragments in links

fixtures/configs/smoketest.toml

@@ -46,7 +46,13 @@ timeout = 20
 retry_wait_time = 2
 
 # Comma-separated list of accepted status codes for valid links.
-accept = [200, 429]
+# Supported values are:
+#
+# accept = ["200..=204", "429"]
+# accept = "200..=204, 429"
+# accept = ["200", "429"]
+# accept = "200, 429"
+accept = ["200", "429"]
 
 # Proceed for server connections considered insecure (invalid TLS).
 insecure = false

lychee-bin/src/client.rs

@@ -26,16 +26,13 @@ pub(crate) fn create(cfg: &Config, cookie_jar: Option<&Arc<CookieStoreMutex>>) -
         cfg.scheme.clone()
     };
 
-    let accepted = match cfg.accept {
-        Some(ref accepted) => {
-            let accepted: Result<HashSet<_>, _> = accepted
-                .iter()
-                .map(|code| StatusCode::from_u16(*code))
-                .collect();
-            Some(accepted?)
-        }
-        None => None,
-    };
+    let accepted = cfg
+        .accept
+        .clone()
+        .into_set()
+        .iter()
+        .map(|value| StatusCode::from_u16(*value))
+        .collect::<Result<HashSet<_>, _>>()?;
 
     // `exclude_mail` will be removed in 1.0. Until then, we need to support it.
     // Therefore, we need to check if both `include_mail` and `exclude_mail` are set to `true`

lychee-bin/src/commands/check.rs

@@ -44,7 +44,7 @@ where
 
     let client = params.client;
     let cache = params.cache;
-    let accept = params.cfg.accept;
+    let accept = params.cfg.accept.into_set();
 
     let pb = if params.cfg.no_progress || params.cfg.verbose.log_level() >= log::Level::Info {
         None
@@ -207,7 +207,7 @@ async fn request_channel_task(
     max_concurrency: usize,
     client: Client,
     cache: Arc<Cache>,
-    accept: Option<HashSet<u16>>,
+    accept: HashSet<u16>,
 ) {
     StreamExt::for_each_concurrent(
         ReceiverStream::new(recv_req),
@@ -230,7 +230,7 @@ async fn handle(
     client: &Client,
     cache: Arc<Cache>,
     request: Request,
-    accept: Option<HashSet<u16>>,
+    accept: HashSet<u16>,
 ) -> Response {
     let uri = request.uri.clone();
     if let Some(v) = cache.get(&uri) {
@@ -244,7 +244,7 @@ async fn handle(
             // `accepted` status codes might have changed from the previous run
             // and they may have an impact on the interpretation of the status
             // code.
-            Status::from_cache_status(v.value().status, accept)
+            Status::from_cache_status(v.value().status, &accept)
         };
         return Response::new(uri.clone(), status, request.source);
     }

lychee-bin/src/options.rs

@@ -1,17 +1,17 @@
 use crate::archive::Archive;
-use crate::parse::{parse_base, parse_statuscodes};
+use crate::parse::parse_base;
 use crate::verbosity::Verbosity;
 use anyhow::{anyhow, Context, Error, Result};
 use clap::{arg, builder::TypedValueParser, Parser};
 use const_format::{concatcp, formatcp};
 use lychee_lib::{
-    Base, BasicAuthSelector, Input, DEFAULT_MAX_REDIRECTS, DEFAULT_MAX_RETRIES,
+    AcceptSelector, Base, BasicAuthSelector, Input, DEFAULT_MAX_REDIRECTS, DEFAULT_MAX_RETRIES,
     DEFAULT_RETRY_WAIT_TIME_SECS, DEFAULT_TIMEOUT_SECS, DEFAULT_USER_AGENT,
 };
 use secrecy::{ExposeSecret, SecretString};
 use serde::Deserialize;
 use std::path::Path;
-use std::{collections::HashSet, fs, path::PathBuf, str::FromStr, time::Duration};
+use std::{fs, path::PathBuf, str::FromStr, time::Duration};
 use strum::VariantNames;
 
 pub(crate) const LYCHEE_IGNORE_FILE: &str = ".lycheeignore";
@@ -91,6 +91,7 @@ default_function! {
     retry_wait_time: usize = DEFAULT_RETRY_WAIT_TIME_SECS;
     method: String = DEFAULT_METHOD.to_string();
     verbosity: Verbosity = Verbosity::default();
+    accept_selector: AcceptSelector = AcceptSelector::default();
 }
 
 // Macro for merging configuration values
@@ -304,10 +305,28 @@ pub(crate) struct Config {
     #[serde(default)]
     pub(crate) header: Vec<String>,
 
-    /// Comma-separated list of accepted status codes for valid links
-    #[arg(short, long, value_parser = parse_statuscodes)]
-    #[serde(default)]
-    pub(crate) accept: Option<HashSet<u16>>,
+    /// A List of accepted status codes for valid links
+    #[arg(
+        short,
+        long,
+        default_value_t,
+        long_help = "A List of accepted status codes for valid links
+
+The following accept range syntax is supported: [start]..[=]end|code. Some valid
+examples are:
+
+- 200..=204
+- 200..204
+- ..=204
+- ..204
+- 200
+
+Use \"lychee --accept '200..=204, 429, 500' <inputs>...\" to provide a comma-
+separated list of accepted status codes. This example will accept 200, 201,
+202, 203, 204, 429, and 500 as valid status codes."
+    )]
+    #[serde(default = "accept_selector")]
+    pub(crate) accept: AcceptSelector,
 
     /// Enable the checking of fragments in links.
     #[arg(long)]
@@ -389,7 +408,8 @@ impl Config {
     pub(crate) fn load_from_file(path: &Path) -> Result<Config> {
         // Read configuration file
         let contents = fs::read_to_string(path)?;
-        toml::from_str(&contents).context("Failed to parse configuration file")
+        toml::from_str(&contents)
+            .map_err(|err| anyhow::anyhow!("Failed to parse configuration file: {}", err))
     }
 
     /// Merge the configuration from TOML into the CLI configuration
@@ -421,7 +441,6 @@ impl Config {
             exclude_mail: false;
             remap: Vec::<String>::new();
             header: Vec::<String>::new();
-            accept: None;
             timeout: DEFAULT_TIMEOUT_SECS;
             retry_wait_time: DEFAULT_RETRY_WAIT_TIME_SECS;
             method: DEFAULT_METHOD;

lychee-bin/src/parse.rs

@@ -1,7 +1,7 @@
 use anyhow::{anyhow, Context, Result};
 use headers::{HeaderMap, HeaderName};
 use lychee_lib::{remap::Remaps, Base};
-use std::{collections::HashSet, time::Duration};
+use std::time::Duration;
 
 /// Split a single HTTP header into a (key, value) tuple
 fn read_header(input: &str) -> Result<(String, String)> {
@@ -40,24 +40,8 @@ pub(crate) fn parse_base(src: &str) -> Result<Base, lychee_lib::ErrorKind> {
     Base::try_from(src)
 }
 
-/// Parse HTTP status codes into a set of `StatusCode`
-///
-/// Note that this function does not convert the status codes into
-/// `StatusCode` but rather into `u16` to avoid the need for
-/// `http` as a dependency and to support custom status codes, which are
-/// necessary for some websites, which don't adhere to the HTTP spec or IANA.
-pub(crate) fn parse_statuscodes(accept: &str) -> Result<HashSet<u16>> {
-    let mut statuscodes = HashSet::new();
-    for code in accept.split(',') {
-        let code: u16 = code.parse::<u16>()?;
-        statuscodes.insert(code);
-    }
-    Ok(statuscodes)
-}
-
 #[cfg(test)]
 mod tests {
-    use std::collections::HashSet;
 
     use headers::HeaderMap;
     use regex::Regex;
@@ -72,13 +56,6 @@ mod tests {
         assert_eq!(parse_headers(&["accept=text/html"]).unwrap(), custom);
     }
 
-    #[test]
-    fn test_parse_statuscodes() {
-        let actual = parse_statuscodes("200,204,301").unwrap();
-        let expected = IntoIterator::into_iter([200, 204, 301]).collect::<HashSet<_>>();
-        assert_eq!(actual, expected);
-    }
-
     #[test]
     fn test_parse_remap() {
         let remaps =

lychee-lib/Cargo.toml

@@ -66,6 +66,8 @@ doc-comment = "0.3.3"
 tempfile = "3.8.0"
 wiremock = "0.5.19"
 serde_json = "1.0.105"
+rstest = "0.18.1"
+toml = "0.7.6"
 
 [features]
 

lychee-lib/src/lib.rs

@@ -91,8 +91,8 @@ pub use crate::{
     collector::Collector,
     filter::{Excludes, Filter, Includes},
     types::{
-        uri::valid::Uri, Base, BasicAuthCredentials, BasicAuthSelector, CacheStatus, CookieJar,
-        ErrorKind, FileType, Input, InputContent, InputSource, Request, Response, ResponseBody,
-        Result, Status,
+        uri::valid::Uri, AcceptRange, AcceptRangeError, AcceptSelector, Base, BasicAuthCredentials,
+        BasicAuthSelector, CacheStatus, CookieJar, ErrorKind, FileType, Input, InputContent,
+        InputSource, Request, Response, ResponseBody, Result, Status,
     },
 };

lychee-lib/src/types/accept/mod.rs

@@ -0,0 +1,5 @@
+mod range;
+mod selector;
+
+pub use range::*;
+pub use selector::*;