diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 558b64e16..b47ad5092 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,16 +91,12 @@ jobs: include: - name: linux / stable - test-features: "--features __internal_proxy_sys_no_cache" - name: linux / beta rust: beta - test-features: "--features __internal_proxy_sys_no_cache" # - name: linux / nightly # rust: nightly - # test-features: "--features __internal_proxy_sys_no_cache" - name: macOS / stable os: macOS-latest - test-features: "--features __internal_proxy_sys_no_cache" - name: windows / stable-x86_64-msvc os: windows-latest @@ -272,7 +268,7 @@ jobs: run: | cargo clean cargo update -Z minimal-versions - cargo update -p proc-macro2 --precise 1.0.60 + cargo update -p proc-macro2 --precise 1.0.87 cargo check cargo check --all-features @@ -300,6 +296,7 @@ jobs: cargo update cargo update -p log --precise 0.4.21 cargo update -p tokio --precise 1.29.1 + cargo update -p tokio-util --precise 0.7.11 cargo update -p url --precise 2.5.0 - uses: Swatinem/rust-cache@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d0860163..0ae3ccc5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,23 @@ -## Unreleased +## v0.12.8 -- Implement `danger_accept_invalid_hostnames` for `rustls`. +- Add support for SOCKS4 proxies. +- Add `multipart::Form::file()` method for adding files easily. +- Add `Body::wrap()` to wrap any `http_body::Body` type. +- Fix the pool configuration to use a timer to remove expired connections. + + +## v0.12.7 + +- Revert adding `impl Service>` for `Client`. + +## v0.12.6 + +- Add support for `danger_accept_invalid_hostnames` for `rustls`. +- Add `impl Service>` for `Client` and `&'_ Client`. +- Add support for `!Sync` bodies in `Body::wrap_stream()`. +- Enable happy eyeballs when `hickory-dns` is used. +- Fix `Proxy` so that `HTTP(S)_PROXY` values take precendence over `ALL_PROXY`. +- Fix `blocking::RequestBuilder::header()` from unsetting `sensitive` on passed header values. ## v0.12.5 diff --git a/Cargo.toml b/Cargo.toml index 6c91f6569..23e6fd656 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "reqwest" -version = "0.12.5" +version = "0.12.8" description = "higher level HTTP client library" keywords = ["http", "request", "client"] categories = ["web-programming::http-client", "wasm"] @@ -143,7 +143,7 @@ rustls = { version = "0.23.4", optional = true, default-features = false, featur rustls-pki-types = { version = "1.1.0", features = ["alloc"] ,optional = true } tokio-rustls = { version = "0.26", optional = true, default-features = false, features = ["tls12"] } webpki-roots = { version = "0.26.0", optional = true } -rustls-native-certs = { version = "0.7", optional = true } +rustls-native-certs = { version = "0.8.0", optional = true } ## cookies cookie_crate = { version = "0.18.0", package = "cookie", optional = true } @@ -154,7 +154,7 @@ async-compression = { version = "0.4.0", default-features = false, features = [" tokio-util = { version = "0.7.9", default-features = false, features = ["codec", "io"], optional = true } ## socks -tokio-socks = { version = "0.5.1", optional = true } +tokio-socks = { version = "0.5.2", optional = true } ## hickory-dns hickory-resolver = { version = "0.24", optional = true, features = ["tokio-runtime"] } @@ -172,7 +172,7 @@ env_logger = "0.11" hyper = { version = "1.1.0", default-features = false, features = ["http1", "http2", "client", "server"] } hyper-util = { version = "0.1.3", features = ["http1", "http2", "client", "client-legacy", "server-auto", "tokio"] } serde = { version = "1.0", features = ["derive"] } -libflate = "1.0" +libflate = "2.1" brotli_crate = { package = "brotli", version = "6.0.0" } zstd_crate = { package = "zstd", version = "0.13" } doc-comment = "0.3" @@ -184,14 +184,14 @@ rustls = { version = "0.23", default-features = false, features = ["ring"] } windows-registry = "0.2" [target.'cfg(target_os = "macos")'.dependencies] -system-configuration = { version = "0.5.1", optional = true } +system-configuration = { version = "0.6.0", optional = true } # wasm [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3.45" serde_json = "1.0" -wasm-bindgen = "0.2.68" +wasm-bindgen = "0.2.89" wasm-bindgen-futures = "0.4.18" wasm-streams = { version = "0.4", optional = true } @@ -216,7 +216,7 @@ features = [ ] [target.'cfg(target_arch = "wasm32")'.dev-dependencies] -wasm-bindgen = { version = "0.2.68", features = ["serde-serialize"] } +wasm-bindgen = { version = "0.2.89", features = ["serde-serialize"] } wasm-bindgen-test = "0.3" [lints.rust] diff --git a/src/async_impl/body.rs b/src/async_impl/body.rs index 8a35585e2..c2f1257c1 100644 --- a/src/async_impl/body.rs +++ b/src/async_impl/body.rs @@ -117,17 +117,6 @@ impl Body { } } - /* - #[cfg(feature = "blocking")] - pub(crate) fn wrap(body: hyper::Body) -> Body { - Body { - inner: Inner::Streaming { - body: Box::pin(WrapHyper(body)), - }, - } - } - */ - pub(crate) fn empty() -> Body { Body::reusable(Bytes::new()) } @@ -138,8 +127,20 @@ impl Body { } } - // pub? - pub(crate) fn streaming(inner: B) -> Body + /// Wrap a [`HttpBody`] in a box inside `Body`. + /// + /// # Example + /// + /// ``` + /// # use reqwest::Body; + /// # use futures_util; + /// # fn main() { + /// let content = "hello,world!".to_string(); + /// + /// let body = Body::wrap(content); + /// # } + /// ``` + pub fn wrap(inner: B) -> Body where B: HttpBody + Send + Sync + 'static, B::Data: Into, @@ -483,7 +484,7 @@ mod tests { assert!(!bytes_body.is_end_stream()); assert_eq!(bytes_body.size_hint().exact(), Some(3)); - let stream_body = Body::streaming(bytes_body); + let stream_body = Body::wrap(bytes_body); assert!(!stream_body.is_end_stream()); assert_eq!(stream_body.size_hint().exact(), None); } diff --git a/src/async_impl/client.rs b/src/async_impl/client.rs index 3bb650244..6e21908a3 100644 --- a/src/async_impl/client.rs +++ b/src/async_impl/client.rs @@ -39,6 +39,8 @@ use crate::dns::{gai::GaiResolver, DnsResolverWithOverrides, DynResolver, Resolv use crate::error; use crate::into_url::try_uri; use crate::redirect::{self, remove_sensitive_headers}; +#[cfg(feature = "__rustls")] +use crate::tls::CertificateRevocationList; #[cfg(feature = "__tls")] use crate::tls::{self, TlsBackend}; #[cfg(feature = "__tls")] @@ -118,6 +120,8 @@ struct Config { tls_built_in_certs_webpki: bool, #[cfg(feature = "rustls-tls-native-roots")] tls_built_in_certs_native: bool, + #[cfg(feature = "__rustls")] + crls: Vec, #[cfg(feature = "__tls")] min_tls_version: Option, #[cfg(feature = "__tls")] @@ -217,6 +221,8 @@ impl ClientBuilder { tls_built_in_certs_native: true, #[cfg(any(feature = "native-tls", feature = "__rustls"))] identity: None, + #[cfg(feature = "__rustls")] + crls: vec![], #[cfg(feature = "__tls")] min_tls_version: None, #[cfg(feature = "__tls")] @@ -514,9 +520,9 @@ impl ClientBuilder { if config.tls_built_in_certs_native { let mut valid_count = 0; let mut invalid_count = 0; - for cert in rustls_native_certs::load_native_certs() - .map_err(crate::error::builder)? - { + + let load_results = rustls_native_certs::load_native_certs(); + for cert in load_results.certs { // Continue on parsing errors, as native stores often include ancient or syntactically // invalid certificates, like root certificates without any X509 extensions. // Inspiration: https://github.com/rustls/rustls/blob/633bf4ba9d9521a95f68766d04c22e2b01e68318/rustls/src/anchors.rs#L105-L112 @@ -529,9 +535,21 @@ impl ClientBuilder { } } if valid_count == 0 && invalid_count > 0 { - return Err(crate::error::builder( - "zero valid certificates found in native root store", - )); + let err = if load_results.errors.is_empty() { + crate::error::builder( + "zero valid certificates found in native root store", + ) + } else { + use std::fmt::Write as _; + let mut acc = String::new(); + for err in load_results.errors { + let _ = writeln!(&mut acc, "{err}"); + } + + crate::error::builder(acc) + }; + + return Err(err); } } @@ -576,9 +594,10 @@ impl ClientBuilder { // Build TLS config let signature_algorithms = provider.signature_verification_algorithms; - let config_builder = rustls::ClientConfig::builder_with_provider(provider) - .with_protocol_versions(&versions) - .map_err(|_| crate::error::builder("invalid TLS versions"))?; + let config_builder = + rustls::ClientConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&versions) + .map_err(|_| crate::error::builder("invalid TLS versions"))?; let config_builder = if !config.certs_verification { config_builder @@ -592,7 +611,26 @@ impl ClientBuilder { signature_algorithms, ))) } else { - config_builder.with_root_certificates(root_cert_store) + if config.crls.is_empty() { + config_builder.with_root_certificates(root_cert_store) + } else { + let crls = config + .crls + .iter() + .map(|e| e.as_rustls_crl()) + .collect::>(); + let verifier = + rustls::client::WebPkiServerVerifier::builder_with_provider( + Arc::new(root_cert_store), + provider, + ) + .with_crls(crls) + .build() + .map_err(|_| { + crate::error::builder("invalid TLS verification settings") + })?; + config_builder.with_webpki_verifier(verifier) + } }; // Finalize TLS config @@ -714,8 +752,8 @@ impl ClientBuilder { } } - #[cfg(not(target_arch = "wasm32"))] builder.timer(hyper_util::rt::TokioTimer::new()); + builder.pool_timer(hyper_util::rt::TokioTimer::new()); builder.pool_idle_timeout(config.pool_idle_timeout); builder.pool_max_idle_per_host(config.pool_max_idle_per_host); connector.set_keepalive(config.tcp_keepalive); @@ -1394,6 +1432,35 @@ impl ClientBuilder { self } + /// Add a certificate revocation list. + /// + /// + /// # Optional + /// + /// This requires the `rustls-tls(-...)` Cargo feature enabled. + #[cfg(feature = "__rustls")] + #[cfg_attr(docsrs, doc(cfg(feature = "rustls-tls")))] + pub fn add_crl(mut self, crl: CertificateRevocationList) -> ClientBuilder { + self.config.crls.push(crl); + self + } + + /// Add multiple certificate revocation lists. + /// + /// + /// # Optional + /// + /// This requires the `rustls-tls(-...)` Cargo feature enabled. + #[cfg(feature = "__rustls")] + #[cfg_attr(docsrs, doc(cfg(feature = "rustls-tls")))] + pub fn add_crls( + mut self, + crls: impl IntoIterator, + ) -> ClientBuilder { + self.config.crls.extend(crls); + self + } + /// Controls the use of built-in/preloaded certificates during certificate validation. /// /// Defaults to `true` -- built-in system certs will be used. @@ -1720,14 +1787,7 @@ impl ClientBuilder { self } - /// Enables the [hickory-dns](hickory_resolver) async resolver instead of a default threadpool - /// using `getaddrinfo`. - /// - /// If the `hickory-dns` feature is turned on, the default option is enabled. - /// - /// # Optional - /// - /// This requires the optional `hickory-dns` feature to be enabled + #[doc(hidden)] #[cfg(feature = "hickory-dns")] #[cfg_attr(docsrs, doc(cfg(feature = "hickory-dns")))] #[deprecated(note = "use `hickory_dns` instead")] @@ -1744,6 +1804,11 @@ impl ClientBuilder { /// # Optional /// /// This requires the optional `hickory-dns` feature to be enabled + /// + /// # Warning + /// + /// The hickory resolver does not work exactly the same, or on all the platforms + /// that the default resolver does #[cfg(feature = "hickory-dns")] #[cfg_attr(docsrs, doc(cfg(feature = "hickory-dns")))] pub fn hickory_dns(mut self, enable: bool) -> ClientBuilder { @@ -1751,22 +1816,10 @@ impl ClientBuilder { self } - /// Disables the hickory-dns async resolver. - /// - /// This method exists even if the optional `hickory-dns` feature is not enabled. - /// This can be used to ensure a `Client` doesn't use the hickory-dns async resolver - /// even if another dependency were to enable the optional `hickory-dns` feature. + #[doc(hidden)] #[deprecated(note = "use `no_hickory_dns` instead")] pub fn no_trust_dns(self) -> ClientBuilder { - #[cfg(feature = "hickory-dns")] - { - self.hickory_dns(false) - } - - #[cfg(not(feature = "hickory-dns"))] - { - self - } + self.no_hickory_dns() } /// Disables the hickory-dns async resolver. diff --git a/src/async_impl/multipart.rs b/src/async_impl/multipart.rs index 75198ca0a..525876dde 100644 --- a/src/async_impl/multipart.rs +++ b/src/async_impl/multipart.rs @@ -3,9 +3,16 @@ use std::borrow::Cow; use std::fmt; use std::pin::Pin; +#[cfg(feature = "stream")] +use std::io; +#[cfg(feature = "stream")] +use std::path::Path; + use bytes::Bytes; use mime_guess::Mime; use percent_encoding::{self, AsciiSet, NON_ALPHANUMERIC}; +#[cfg(feature = "stream")] +use tokio::fs::File; use futures_core::Stream; use futures_util::{future, stream, StreamExt}; @@ -82,6 +89,33 @@ impl Form { self.part(name, Part::text(value)) } + /// Adds a file field. + /// + /// The path will be used to try to guess the filename and mime. + /// + /// # Examples + /// + /// ```no_run + /// # async fn run() -> std::io::Result<()> { + /// let form = reqwest::multipart::Form::new() + /// .file("key", "/path/to/file").await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// # Errors + /// + /// Errors when the file cannot be opened. + #[cfg(feature = "stream")] + #[cfg_attr(docsrs, doc(cfg(feature = "stream")))] + pub async fn file(self, name: T, path: U) -> io::Result
+ where + T: Into>, + U: AsRef, + { + Ok(self.part(name, Part::file(path).await?)) + } + /// Adds a customized Part. pub fn part(self, name: T, part: Part) -> Form where @@ -218,6 +252,30 @@ impl Part { Part::new(value.into(), Some(length)) } + /// Makes a file parameter. + /// + /// # Errors + /// + /// Errors when the file cannot be opened. + #[cfg(feature = "stream")] + #[cfg_attr(docsrs, doc(cfg(feature = "stream")))] + pub async fn file>(path: T) -> io::Result { + let path = path.as_ref(); + let file_name = path + .file_name() + .map(|filename| filename.to_string_lossy().into_owned()); + let ext = path.extension().and_then(|ext| ext.to_str()).unwrap_or(""); + let mime = mime_guess::from_ext(ext).first_or_octet_stream(); + let file = File::open(path).await?; + let field = Part::stream(file).mime(mime); + + Ok(if let Some(file_name) = file_name { + field.file_name(file_name) + } else { + field + }) + } + fn new(value: Body, body_length: Option) -> Part { Part { meta: PartMetadata::new(), diff --git a/src/async_impl/request.rs b/src/async_impl/request.rs index aa900ca02..76b40a788 100644 --- a/src/async_impl/request.rs +++ b/src/async_impl/request.rs @@ -309,6 +309,9 @@ impl RequestBuilder { /// # Ok(()) /// # } /// ``` + /// + /// In additional the request's body, the Content-Type and Content-Length fields are + /// appropriately set. #[cfg(feature = "multipart")] #[cfg_attr(docsrs, doc(cfg(feature = "multipart")))] pub fn multipart(self, mut multipart: multipart::Form) -> RequestBuilder { diff --git a/src/async_impl/response.rs b/src/async_impl/response.rs index 17be37030..23e30d3ed 100644 --- a/src/async_impl/response.rs +++ b/src/async_impl/response.rs @@ -442,7 +442,7 @@ impl fmt::Debug for Response { /// A `Response` can be piped as the `Body` of another request. impl From for Body { fn from(r: Response) -> Body { - Body::streaming(r.res.into_body()) + Body::wrap(r.res.into_body()) } } @@ -477,7 +477,7 @@ impl> From> for Response { impl From for http::Response { fn from(r: Response) -> http::Response { let (parts, body) = r.res.into_parts(); - let body = Body::streaming(body); + let body = Body::wrap(body); http::Response::from_parts(parts, body) } } diff --git a/src/blocking/client.rs b/src/blocking/client.rs index d4b973ee6..9a447e2d5 100644 --- a/src/blocking/client.rs +++ b/src/blocking/client.rs @@ -19,6 +19,8 @@ use super::wait; use crate::dns::Resolve; #[cfg(feature = "__tls")] use crate::tls; +#[cfg(feature = "__rustls")] +use crate::tls::CertificateRevocationList; #[cfg(feature = "__tls")] use crate::Certificate; #[cfg(any(feature = "native-tls", feature = "__rustls"))] @@ -606,6 +608,33 @@ impl ClientBuilder { self.with_inner(move |inner| inner.add_root_certificate(cert)) } + /// Add a certificate revocation list. + /// + /// + /// # Optional + /// + /// This requires the `rustls-tls(-...)` Cargo feature enabled. + #[cfg(feature = "__rustls")] + #[cfg_attr(docsrs, doc(cfg(feature = "rustls-tls")))] + pub fn add_crl(mut self, crl: CertificateRevocationList) -> ClientBuilder { + self.with_inner(move |inner| inner.add_crl(crl)) + } + + /// Add multiple certificate revocation lists. + /// + /// + /// # Optional + /// + /// This requires the `rustls-tls(-...)` Cargo feature enabled. + #[cfg(feature = "__rustls")] + #[cfg_attr(docsrs, doc(cfg(feature = "rustls-tls")))] + pub fn add_crls( + mut self, + crls: impl IntoIterator, + ) -> ClientBuilder { + self.with_inner(move |inner| inner.add_crls(crls)) + } + /// Controls the use of built-in system certificates during certificate validation. /// /// Defaults to `true` -- built-in system certs will be used. diff --git a/src/blocking/multipart.rs b/src/blocking/multipart.rs index 8f7c7bc84..4f18a2aae 100644 --- a/src/blocking/multipart.rs +++ b/src/blocking/multipart.rs @@ -104,7 +104,7 @@ impl Form { /// /// ```no_run /// # fn run() -> std::io::Result<()> { - /// let files = reqwest::blocking::multipart::Form::new() + /// let form = reqwest::blocking::multipart::Form::new() /// .file("key", "/path/to/file")?; /// # Ok(()) /// # } diff --git a/src/connect.rs b/src/connect.rs index ff76c57f8..bf681f9b5 100644 --- a/src/connect.rs +++ b/src/connect.rs @@ -206,6 +206,7 @@ impl Connector { #[cfg(feature = "socks")] async fn connect_socks(&self, dst: Uri, proxy: ProxyScheme) -> Result { let dns = match proxy { + ProxyScheme::Socks4 { .. } => socks::DnsResolve::Local, ProxyScheme::Socks5 { remote_dns: false, .. } => socks::DnsResolve::Local, @@ -367,6 +368,8 @@ impl Connector { ProxyScheme::Http { host, auth } => (into_uri(Scheme::HTTP, host), auth), ProxyScheme::Https { host, auth } => (into_uri(Scheme::HTTPS, host), auth), #[cfg(feature = "socks")] + ProxyScheme::Socks4 { .. } => return self.connect_socks(dst, proxy_scheme).await, + #[cfg(feature = "socks")] ProxyScheme::Socks5 { .. } => return self.connect_socks(dst, proxy_scheme).await, }; @@ -1031,7 +1034,7 @@ mod socks { use http::Uri; use tokio::net::TcpStream; - use tokio_socks::tcp::Socks5Stream; + use tokio_socks::tcp::{Socks4Stream, Socks5Stream}; use super::{BoxError, Scheme}; use crate::proxy::ProxyScheme; @@ -1064,28 +1067,33 @@ mod socks { } } - let (socket_addr, auth) = match proxy { - ProxyScheme::Socks5 { addr, auth, .. } => (addr, auth), - _ => unreachable!(), - }; - - // Get a Tokio TcpStream - let stream = if let Some((username, password)) = auth { - Socks5Stream::connect_with_password( - socket_addr, - (host.as_str(), port), - &username, - &password, - ) - .await - .map_err(|e| format!("socks connect error: {e}"))? - } else { - Socks5Stream::connect(socket_addr, (host.as_str(), port)) - .await - .map_err(|e| format!("socks connect error: {e}"))? - }; + match proxy { + ProxyScheme::Socks4 { addr } => { + let stream = Socks4Stream::connect(addr, (host.as_str(), port)) + .await + .map_err(|e| format!("socks connect error: {e}"))?; + Ok(stream.into_inner()) + } + ProxyScheme::Socks5 { addr, ref auth, .. } => { + let stream = if let Some((username, password)) = auth { + Socks5Stream::connect_with_password( + addr, + (host.as_str(), port), + &username, + &password, + ) + .await + .map_err(|e| format!("socks connect error: {e}"))? + } else { + Socks5Stream::connect(addr, (host.as_str(), port)) + .await + .map_err(|e| format!("socks connect error: {e}"))? + }; - Ok(stream.into_inner()) + Ok(stream.into_inner()) + } + _ => unreachable!(), + } } } diff --git a/src/dns/hickory.rs b/src/dns/hickory.rs index 44b943827..a94160b2d 100644 --- a/src/dns/hickory.rs +++ b/src/dns/hickory.rs @@ -1,11 +1,12 @@ //! DNS resolution via the [hickory-resolver](https://github.com/hickory-dns/hickory-dns) crate use hickory_resolver::{ - config::LookupIpStrategy, lookup_ip::LookupIpIntoIter, system_conf, TokioAsyncResolver, + config::LookupIpStrategy, error::ResolveError, lookup_ip::LookupIpIntoIter, system_conf, + TokioAsyncResolver, }; use once_cell::sync::OnceCell; -use std::io; +use std::fmt; use std::net::SocketAddr; use std::sync::Arc; @@ -24,6 +25,9 @@ struct SocketAddrs { iter: LookupIpIntoIter, } +#[derive(Debug)] +struct HickoryDnsSystemConfError(ResolveError); + impl Resolve for HickoryDnsResolver { fn resolve(&self, name: Name) -> Resolving { let resolver = self.clone(); @@ -51,13 +55,20 @@ impl Iterator for SocketAddrs { /// which reads from `/etc/resolve.conf`. The options are /// overridden to look up for both IPv4 and IPv6 addresses /// to work with "happy eyeballs" algorithm. -fn new_resolver() -> io::Result { - let (config, mut opts) = system_conf::read_system_conf().map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("error reading DNS system conf: {e}"), - ) - })?; +fn new_resolver() -> Result { + let (config, mut opts) = system_conf::read_system_conf().map_err(HickoryDnsSystemConfError)?; opts.ip_strategy = LookupIpStrategy::Ipv4AndIpv6; Ok(TokioAsyncResolver::tokio(config, opts)) } + +impl fmt::Display for HickoryDnsSystemConfError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("error reading DNS system conf for hickory-dns") + } +} + +impl std::error::Error for HickoryDnsSystemConfError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } +} diff --git a/src/proxy.rs b/src/proxy.rs index 5be207a8a..61d1202bd 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -7,7 +7,6 @@ use crate::into_url::{IntoUrl, IntoUrlSealed}; use crate::Url; use http::{header::HeaderValue, Uri}; use ipnet::IpNet; -use once_cell::sync::Lazy; use percent_encoding::percent_decode; use std::collections::HashMap; use std::env; @@ -106,6 +105,8 @@ pub enum ProxyScheme { host: http::uri::Authority, }, #[cfg(feature = "socks")] + Socks4 { addr: SocketAddr }, + #[cfg(feature = "socks")] Socks5 { addr: SocketAddr, auth: Option<(String, String)>, @@ -278,13 +279,9 @@ impl Proxy { } pub(crate) fn system() -> Proxy { - let mut proxy = if cfg!(feature = "__internal_proxy_sys_no_cache") { - Proxy::new(Intercept::System(Arc::new(get_sys_proxies( - get_from_platform(), - )))) - } else { - Proxy::new(Intercept::System(SYS_PROXIES.clone())) - }; + let mut proxy = Proxy::new(Intercept::System(Arc::new(get_sys_proxies( + get_from_platform(), + )))); proxy.no_proxy = NoProxy::from_env(); proxy } @@ -577,6 +574,16 @@ impl ProxyScheme { }) } + /// Proxy traffic via the specified socket address over SOCKS4 + /// + /// # Note + /// + /// Current SOCKS4 support is provided via blocking IO. + #[cfg(feature = "socks")] + fn socks4(addr: SocketAddr) -> crate::Result { + Ok(ProxyScheme::Socks4 { addr }) + } + /// Proxy traffic via the specified socket address over SOCKS5 /// /// # Note @@ -628,6 +635,10 @@ impl ProxyScheme { *auth = Some(header); } #[cfg(feature = "socks")] + ProxyScheme::Socks4 { .. } => { + panic!("Socks4 is not supported for this method") + } + #[cfg(feature = "socks")] ProxyScheme::Socks5 { ref mut auth, .. } => { *auth = Some((username.into(), password.into())); } @@ -643,8 +654,12 @@ impl ProxyScheme { *auth = Some(header_value); } #[cfg(feature = "socks")] + ProxyScheme::Socks4 { .. } => { + panic!("Socks4 is not supported for this method") + } + #[cfg(feature = "socks")] ProxyScheme::Socks5 { .. } => { - panic!("Socks is not supported for this method") + panic!("Socks5 is not supported for this method") } } } @@ -662,6 +677,8 @@ impl ProxyScheme { } } #[cfg(feature = "socks")] + ProxyScheme::Socks4 { .. } => {} + #[cfg(feature = "socks")] ProxyScheme::Socks5 { .. } => {} } @@ -670,7 +687,7 @@ impl ProxyScheme { /// Convert a URL into a proxy scheme /// - /// Supported schemes: HTTP, HTTPS, (SOCKS5, SOCKS5H if `socks` feature is enabled). + /// Supported schemes: HTTP, HTTPS, (SOCKS4, SOCKS5, SOCKS5H if `socks` feature is enabled). // Private for now... fn parse(url: Url) -> crate::Result { use url::Position; @@ -680,7 +697,7 @@ impl ProxyScheme { let to_addr = || { let addrs = url .socket_addrs(|| match url.scheme() { - "socks5" | "socks5h" => Some(1080), + "socks4" | "socks5" | "socks5h" => Some(1080), _ => None, }) .map_err(crate::error::builder)?; @@ -694,6 +711,8 @@ impl ProxyScheme { "http" => Self::http(&url[Position::BeforeHost..Position::AfterPort])?, "https" => Self::https(&url[Position::BeforeHost..Position::AfterPort])?, #[cfg(feature = "socks")] + "socks4" => Self::socks4(to_addr()?)?, + #[cfg(feature = "socks")] "socks5" => Self::socks5(to_addr()?)?, #[cfg(feature = "socks")] "socks5h" => Self::socks5h(to_addr()?)?, @@ -715,6 +734,8 @@ impl ProxyScheme { ProxyScheme::Http { .. } => "http", ProxyScheme::Https { .. } => "https", #[cfg(feature = "socks")] + ProxyScheme::Socks4 { .. } => "socks4", + #[cfg(feature = "socks")] ProxyScheme::Socks5 { .. } => "socks5", } } @@ -725,6 +746,8 @@ impl ProxyScheme { ProxyScheme::Http { host, .. } => host.as_str(), ProxyScheme::Https { host, .. } => host.as_str(), #[cfg(feature = "socks")] + ProxyScheme::Socks4 { .. } => panic!("socks4"), + #[cfg(feature = "socks")] ProxyScheme::Socks5 { .. } => panic!("socks5"), } } @@ -736,6 +759,10 @@ impl fmt::Debug for ProxyScheme { ProxyScheme::Http { auth: _auth, host } => write!(f, "http://{host}"), ProxyScheme::Https { auth: _auth, host } => write!(f, "https://{host}"), #[cfg(feature = "socks")] + ProxyScheme::Socks4 { addr } => { + write!(f, "socks4://{addr}") + } + #[cfg(feature = "socks")] ProxyScheme::Socks5 { addr, auth: _auth, @@ -844,9 +871,6 @@ impl Dst for Uri { } } -static SYS_PROXIES: Lazy> = - Lazy::new(|| Arc::new(get_sys_proxies(get_from_platform()))); - /// Get system proxies information. /// /// All platforms will check for proxy settings via environment variables. diff --git a/src/tls.rs b/src/tls.rs index 83f3feee8..e5480dc96 100644 --- a/src/tls.rs +++ b/src/tls.rs @@ -58,6 +58,13 @@ use std::{ io::{BufRead, BufReader}, }; +/// Represents a X509 certificate revocation list. +#[cfg(feature = "__rustls")] +pub struct CertificateRevocationList { + #[cfg(feature = "__rustls")] + inner: rustls_pki_types::CertificateRevocationListDer<'static>, +} + /// Represents a server X509 certificate. #[derive(Clone)] pub struct Certificate { @@ -409,6 +416,75 @@ impl Identity { } } +#[cfg(feature = "__rustls")] +impl CertificateRevocationList { + /// Parses a PEM encoded CRL. + /// + /// # Examples + /// + /// ``` + /// # use std::fs::File; + /// # use std::io::Read; + /// # fn crl() -> Result<(), Box> { + /// let mut buf = Vec::new(); + /// File::open("my_crl.pem")? + /// .read_to_end(&mut buf)?; + /// let crl = reqwest::tls::CertificateRevocationList::from_pem(&buf)?; + /// # drop(crl); + /// # Ok(()) + /// # } + /// ``` + /// + /// # Optional + /// + /// This requires the `rustls-tls(-...)` Cargo feature enabled. + #[cfg(feature = "__rustls")] + pub fn from_pem(pem: &[u8]) -> crate::Result { + Ok(CertificateRevocationList { + #[cfg(feature = "__rustls")] + inner: rustls_pki_types::CertificateRevocationListDer::from(pem.to_vec()), + }) + } + + /// Creates a collection of `CertificateRevocationList`s from a PEM encoded CRL bundle. + /// Example byte sources may be `.crl` or `.pem` files. + /// + /// # Examples + /// + /// ``` + /// # use std::fs::File; + /// # use std::io::Read; + /// # fn crls() -> Result<(), Box> { + /// let mut buf = Vec::new(); + /// File::open("crl-bundle.crl")? + /// .read_to_end(&mut buf)?; + /// let crls = reqwest::tls::CertificateRevocationList::from_pem_bundle(&buf)?; + /// # drop(crls); + /// # Ok(()) + /// # } + /// ``` + /// + /// # Optional + /// + /// This requires the `rustls-tls(-...)` Cargo feature enabled. + #[cfg(feature = "__rustls")] + pub fn from_pem_bundle(pem_bundle: &[u8]) -> crate::Result> { + let mut reader = BufReader::new(pem_bundle); + + rustls_pemfile::crls(&mut reader) + .map(|result| match result { + Ok(crl) => Ok(CertificateRevocationList { inner: crl }), + Err(_) => Err(crate::error::builder("invalid crl encoding")), + }) + .collect::>>() + } + + #[cfg(feature = "__rustls")] + pub(crate) fn as_rustls_crl<'a>(&self) -> rustls_pki_types::CertificateRevocationListDer<'a> { + self.inner.clone() + } +} + impl fmt::Debug for Certificate { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("Certificate").finish() @@ -421,6 +497,13 @@ impl fmt::Debug for Identity { } } +#[cfg(feature = "__rustls")] +impl fmt::Debug for CertificateRevocationList { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("CertificateRevocationList").finish() + } +} + /// A TLS protocol version. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct Version(InnerVersion); @@ -736,4 +819,24 @@ mod tests { assert!(Certificate::from_pem_bundle(PEM_BUNDLE).is_ok()) } + + #[cfg(feature = "__rustls")] + #[test] + fn crl_from_pem() { + let pem = b"-----BEGIN X509 CRL-----\n-----END X509 CRL-----\n"; + + CertificateRevocationList::from_pem(pem).unwrap(); + } + + #[cfg(feature = "__rustls")] + #[test] + fn crl_from_pem_bundle() { + let pem_bundle = std::fs::read("tests/support/crl.pem").unwrap(); + + let result = CertificateRevocationList::from_pem_bundle(&pem_bundle); + + assert!(result.is_ok()); + let result = result.unwrap(); + assert_eq!(result.len(), 1); + } } diff --git a/tests/client.rs b/tests/client.rs index 18aaf4e99..51fb9dfa0 100644 --- a/tests/client.rs +++ b/tests/client.rs @@ -572,3 +572,24 @@ async fn highly_concurrent_requests_to_slow_http2_server_with_low_max_concurrent server.shutdown().await; } + +#[tokio::test] +async fn close_connection_after_idle_timeout() { + let mut server = server::http(move |_| async move { http::Response::default() }); + + let client = reqwest::Client::builder() + .pool_idle_timeout(std::time::Duration::from_secs(1)) + .build() + .unwrap(); + + let url = format!("http://{}", server.addr()); + + client.get(&url).send().await.unwrap(); + + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + + assert!(server + .events() + .iter() + .any(|e| matches!(e, server::Event::ConnectionClosed))); +} diff --git a/tests/multipart.rs b/tests/multipart.rs index 8b5149e1d..bac6da314 100644 --- a/tests/multipart.rs +++ b/tests/multipart.rs @@ -175,3 +175,58 @@ fn blocking_file_part() { assert_eq!(res.url().as_str(), &url); assert_eq!(res.status(), reqwest::StatusCode::OK); } + +#[cfg(feature = "stream")] +#[tokio::test] +async fn async_impl_file_part() { + let _ = env_logger::try_init(); + + let form = reqwest::multipart::Form::new() + .file("foo", "Cargo.lock") + .await + .unwrap(); + + let fcontents = std::fs::read_to_string("Cargo.lock").unwrap(); + + let expected_body = format!( + "\ + --{0}\r\n\ + Content-Disposition: form-data; name=\"foo\"; filename=\"Cargo.lock\"\r\n\ + Content-Type: application/octet-stream\r\n\r\n\ + {1}\r\n\ + --{0}--\r\n\ + ", + form.boundary(), + fcontents + ); + + let ct = format!("multipart/form-data; boundary={}", form.boundary()); + + let server = server::http(move |req| { + let ct = ct.clone(); + let expected_body = expected_body.clone(); + async move { + assert_eq!(req.method(), "POST"); + assert_eq!(req.headers()["content-type"], ct); + assert_eq!(req.headers()["transfer-encoding"], "chunked"); + + let full = req.collect().await.unwrap().to_bytes(); + + assert_eq!(full, expected_body.as_bytes()); + + http::Response::default() + } + }); + + let url = format!("http://{}/multipart/3", server.addr()); + + let res = reqwest::Client::new() + .post(&url) + .multipart(form) + .send() + .await + .unwrap(); + + assert_eq!(res.url().as_str(), &url); + assert_eq!(res.status(), reqwest::StatusCode::OK); +} diff --git a/tests/proxy.rs b/tests/proxy.rs index 9231a3267..25d0f615c 100644 --- a/tests/proxy.rs +++ b/tests/proxy.rs @@ -163,7 +163,6 @@ async fn test_no_proxy() { assert_eq!(res.status(), reqwest::StatusCode::OK); } -#[cfg_attr(not(feature = "__internal_proxy_sys_no_cache"), ignore)] #[tokio::test] async fn test_using_system_proxy() { let url = "http://not.a.real.sub.hyper.rs/prox"; @@ -175,9 +174,6 @@ async fn test_using_system_proxy() { async { http::Response::default() } }); - // Note: we're relying on the `__internal_proxy_sys_no_cache` feature to - // check the environment every time. - // save system setting first. let system_proxy = env::var("http_proxy"); // set-up http proxy. diff --git a/tests/support/crl.pem b/tests/support/crl.pem new file mode 100644 index 000000000..190f2c7c6 --- /dev/null +++ b/tests/support/crl.pem @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBnjCBhwIBATANBgkqhkiG9w0BAQsFADANMQswCQYDVQQDDAJjYRcNMjQwOTI2 +MDA0MjU1WhcNMjQxMDI2MDA0MjU1WjAUMBICAQEXDTI0MDkyNjAwNDI0NlqgMDAu +MB8GA1UdIwQYMBaAFDxOaZI8zUaGX7mXAZ9Zd8jhyC3sMAsGA1UdFAQEAgIQATAN +BgkqhkiG9w0BAQsFAAOCAQEAsqBa289UYKAOaH2gp3yC7YBF7uVZ25i3WV/InKjK +zT/fFzZ9rL87ofl0VuR0GPAfwLXFQ96vYUg/nrlxF/A6FmQKf9JSlVBIVXaS2uyk +fmdVX8fdU13uD2uKThT5Fojk5nKAeui0xwjTHqe9BjyDscQ5d5pkLIJUj/JbQmRF +D/OtEpYQZMAdHLDF0a/9v69g/evlPlpTcikAU+T8rXp45rrsuuUgyhJ00UnE41j8 +MmMi3cn23JjFTyOrYx5g/0VFUNcwZpgZSnxNvFbcoh9oHHqS+UDESrwQmkmwrVvH +a7PEJq5ZPtjUPa0i7oFNa9cC+11Doo5bxkpCWhypvgTUzw== +-----END X509 CRL----- diff --git a/tests/support/server.rs b/tests/support/server.rs index 43742b60e..29835ead1 100644 --- a/tests/support/server.rs +++ b/tests/support/server.rs @@ -12,13 +12,27 @@ use tokio::sync::oneshot; pub struct Server { addr: net::SocketAddr, panic_rx: std_mpsc::Receiver<()>, + events_rx: std_mpsc::Receiver, shutdown_tx: Option>, } +#[non_exhaustive] +pub enum Event { + ConnectionClosed, +} + impl Server { pub fn addr(&self) -> net::SocketAddr { self.addr } + + pub fn events(&mut self) -> Vec { + let mut events = Vec::new(); + while let Ok(event) = self.events_rx.try_recv() { + events.push(event); + } + events + } } impl Drop for Server { @@ -67,6 +81,7 @@ where let (shutdown_tx, mut shutdown_rx) = oneshot::channel(); let (panic_tx, panic_rx) = std_mpsc::channel(); + let (events_tx, events_rx) = std_mpsc::channel(); let tname = format!( "test({})-support-server", test_name, @@ -92,8 +107,10 @@ where async move { Ok::<_, Infallible>(fut.await) } }); let builder = builder.clone(); + let events_tx = events_tx.clone(); tokio::spawn(async move { let _ = builder.serve_connection_with_upgrades(hyper_util::rt::TokioIo::new(io), svc).await; + let _ = events_tx.send(Event::ConnectionClosed); }); } } @@ -105,6 +122,7 @@ where Server { addr, panic_rx, + events_rx, shutdown_tx: Some(shutdown_tx), } }) @@ -152,6 +170,7 @@ where let (shutdown_tx, mut shutdown_rx) = oneshot::channel(); let (panic_tx, panic_rx) = std_mpsc::channel(); + let (events_tx, events_rx) = std_mpsc::channel(); let tname = format!( "test({})-support-server", test_name, @@ -169,9 +188,11 @@ where Some(accepted) = endpoint.accept() => { let conn = accepted.await.expect("accepted"); let mut h3_conn = h3::server::Connection::new(h3_quinn::Connection::new(conn)).await.unwrap(); + let events_tx = events_tx.clone(); let func = func.clone(); tokio::spawn(async move { while let Ok(Some((req, stream))) = h3_conn.accept().await { + let events_tx = events_tx.clone(); let func = func.clone(); tokio::spawn(async move { let (mut tx, rx) = stream.split(); @@ -198,6 +219,7 @@ where } } tx.finish().await.unwrap(); + events_tx.send(Event::ConnectionClosed).unwrap(); }); } }); @@ -211,6 +233,7 @@ where Server { addr, panic_rx, + events_rx, shutdown_tx: Some(shutdown_tx), } }) diff --git a/tests/timeouts.rs b/tests/timeouts.rs index c3649ea9f..79a6fbb4d 100644 --- a/tests/timeouts.rs +++ b/tests/timeouts.rs @@ -19,6 +19,7 @@ async fn client_timeout() { let client = reqwest::Client::builder() .timeout(Duration::from_millis(100)) + .no_proxy() .build() .unwrap(); @@ -44,7 +45,7 @@ async fn request_timeout() { } }); - let client = reqwest::Client::builder().build().unwrap(); + let client = reqwest::Client::builder().no_proxy().build().unwrap(); let url = format!("http://{}/slow", server.addr()); @@ -71,10 +72,11 @@ async fn connect_timeout() { let client = reqwest::Client::builder() .connect_timeout(Duration::from_millis(100)) + .no_proxy() .build() .unwrap(); - let url = "http://10.255.255.1:81/slow"; + let url = "http://192.0.2.1:81/slow"; let res = client .get(url) @@ -98,9 +100,10 @@ async fn connect_many_timeout_succeeds() { let client = reqwest::Client::builder() .resolve_to_addrs( "many_addrs", - &["10.255.255.1:81".parse().unwrap(), server.addr()], + &["192.0.2.1:81".parse().unwrap(), server.addr()], ) .connect_timeout(Duration::from_millis(100)) + .no_proxy() .build() .unwrap(); @@ -123,11 +126,12 @@ async fn connect_many_timeout() { .resolve_to_addrs( "many_addrs", &[ - "10.255.255.1:81".parse().unwrap(), - "10.255.255.2:81".parse().unwrap(), + "192.0.2.1:81".parse().unwrap(), + "192.0.2.2:81".parse().unwrap(), ], ) .connect_timeout(Duration::from_millis(100)) + .no_proxy() .build() .unwrap(); @@ -190,6 +194,7 @@ async fn read_timeout_applies_to_headers() { let client = reqwest::Client::builder() .read_timeout(Duration::from_millis(100)) + .no_proxy() .build() .unwrap(); @@ -410,7 +415,7 @@ async fn response_body_timeout_forwards_size_hint() { let server = server::http(move |_req| async { http::Response::new(b"hello".to_vec().into()) }); - let client = reqwest::Client::new(); + let client = reqwest::Client::builder().no_proxy().build().unwrap(); let url = format!("http://{}/slow", server.addr());