Skip to content

feat(cli): add custom dns resolver option #587

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
May 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 51 additions & 60 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ serde = "1.0.124"
serde_derive = "1.0.116"
cidr-utils = "0.5.1"
itertools = "0.9.0"
trust-dns-resolver = { version = "0.23.2", features = ["dns-over-rustls"] }
hickory-resolver = { version = "0.24.0", features = ["dns-over-rustls"] }
anyhow = "1.0.40"
subprocess = "0.2.6"
text_placeholder = { version = "0.5", features = ["struct_context"] }
Expand Down
100 changes: 87 additions & 13 deletions src/address.rs
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
//! Provides functions to parse input IP addresses, CIDRs or files.
use std::fs::File;
use std::fs::{self, File};
use std::io::{prelude::*, BufReader};
use std::net::{IpAddr, ToSocketAddrs};
use std::net::{IpAddr, SocketAddr, ToSocketAddrs};
use std::path::Path;
use std::str::FromStr;

use cidr_utils::cidr::IpCidr;
use log::debug;
use trust_dns_resolver::{
config::{ResolverConfig, ResolverOpts},
use hickory_resolver::{
config::{NameServerConfig, Protocol, ResolverConfig, ResolverOpts},
Resolver,
};
use log::debug;

use crate::input::Opts;
use crate::warning;
Expand All @@ -29,8 +30,7 @@ use crate::warning;
pub fn parse_addresses(input: &Opts) -> Vec<IpAddr> {
let mut ips: Vec<IpAddr> = Vec::new();
let mut unresolved_addresses: Vec<&str> = Vec::new();
let backup_resolver =
Resolver::new(ResolverConfig::cloudflare_tls(), ResolverOpts::default()).unwrap();
let backup_resolver = get_resolver(&input.resolver);

for address in &input.addresses {
let parsed_ips = parse_address(address, &backup_resolver);
Expand Down Expand Up @@ -72,11 +72,14 @@ pub fn parse_addresses(input: &Opts) -> Vec<IpAddr> {
/// Given a string, parse it as a host, IP address, or CIDR.
///
/// This allows us to pass files as hosts or cidr or IPs easily
/// Call this every time you have a possible IP_or_host
/// Call this every time you have a possible IP-or-host.
///
/// If the address is a domain, we can self-resolve the domain locally
/// or resolve it by dns resolver list.
///
/// ```rust
/// # use rustscan::address::parse_address;
/// # use trust_dns_resolver::Resolver;
/// # use hickory_resolver::Resolver;
/// let ips = parse_address("127.0.0.1", &Resolver::default().unwrap());
/// ```
pub fn parse_address(address: &str, resolver: &Resolver) -> Vec<IpAddr> {
Expand All @@ -94,7 +97,7 @@ pub fn parse_address(address: &str, resolver: &Resolver) -> Vec<IpAddr> {

/// Uses DNS to get the IPS associated with host
fn resolve_ips_from_host(source: &str, backup_resolver: &Resolver) -> Vec<IpAddr> {
let mut ips: Vec<std::net::IpAddr> = Vec::new();
let mut ips: Vec<IpAddr> = Vec::new();

if let Ok(addrs) = source.to_socket_addrs() {
for ip in addrs {
Expand All @@ -107,16 +110,64 @@ fn resolve_ips_from_host(source: &str, backup_resolver: &Resolver) -> Vec<IpAddr
ips
}

/// Derive a DNS resolver.
///
/// 1. if the `resolver` parameter has been set:
/// 1. assume the parameter is a path and attempt to read IPs.
/// 2. parse the input as a comma-separated list of IPs.
/// 2. if `resolver` is not set:
/// 1. attempt to derive a resolver from the system config. (e.g.
/// `/etc/resolv.conf` on *nix).
/// 2. finally, build a CloudFlare-based resolver (default
/// behaviour).
fn get_resolver(resolver: &Option<String>) -> Resolver {
match resolver {
Some(r) => {
let mut config = ResolverConfig::new();
let resolver_ips = match read_resolver_from_file(r) {
Ok(ips) => ips,
Err(_) => r
.split(',')
.filter_map(|r| IpAddr::from_str(r).ok())
.collect::<Vec<_>>(),
};
for ip in resolver_ips {
config.add_name_server(NameServerConfig::new(
SocketAddr::new(ip, 53),
Protocol::Udp,
));
}
Resolver::new(config, ResolverOpts::default()).unwrap()
}
None => match Resolver::from_system_conf() {
Ok(resolver) => resolver,
Err(_) => {
Resolver::new(ResolverConfig::cloudflare_tls(), ResolverOpts::default()).unwrap()
}
},
}
}

/// Parses and input file of IPs for use in DNS resolution.
fn read_resolver_from_file(path: &str) -> Result<Vec<IpAddr>, std::io::Error> {
let ips = fs::read_to_string(path)?
.lines()
.filter_map(|line| IpAddr::from_str(line.trim()).ok())
.collect();

Ok(ips)
}

#[cfg(not(tarpaulin_include))]
/// Parses an input file of IPs and uses those
fn read_ips_from_file(
ips: &std::path::Path,
backup_resolver: &Resolver,
) -> Result<Vec<std::net::IpAddr>, std::io::Error> {
) -> Result<Vec<IpAddr>, std::io::Error> {
let file = File::open(ips)?;
let reader = BufReader::new(file);

let mut ips: Vec<std::net::IpAddr> = Vec::new();
let mut ips: Vec<IpAddr> = Vec::new();

for address_line in reader.lines() {
if let Ok(address) = address_line {
Expand All @@ -131,7 +182,7 @@ fn read_ips_from_file(

#[cfg(test)]
mod tests {
use super::{parse_addresses, Opts};
use super::{get_resolver, parse_addresses, Opts};
use std::net::Ipv4Addr;

#[test]
Expand Down Expand Up @@ -204,4 +255,27 @@ mod tests {
let ips = parse_addresses(&opts);
assert_eq!(ips.len(), 0);
}

#[test]
fn resolver_default_cloudflare() {
let opts = Opts::default();

let resolver = get_resolver(&opts.resolver);
let lookup = resolver.lookup_ip("www.example.com.").unwrap();

assert!(opts.resolver.is_none());
assert!(lookup.iter().next().is_some());
}

#[test]
fn resolver_args_google_dns() {
let mut opts = Opts::default();
// https://developers.google.com/speed/public-dns
opts.resolver = Some("8.8.8.8,8.8.4.4".to_owned());

let resolver = get_resolver(&opts.resolver);
let lookup = resolver.lookup_ip("www.example.com.").unwrap();

assert!(lookup.iter().next().is_some());
}
Comment on lines +259 to +280
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we try to avoid network tests so you can test on machines without the internet, in this case mocking the test output 🤔

But I think for the things which require this (Homebrew requires no networked tests) we have our own different tests specific to it

Although generally I regualrly make the argument that you cannot test a network CLI tool without some kind of network 😂 so approved!

Copy link
Collaborator Author

@PsypherPunk PsypherPunk May 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, I mulled this one over for a while: the only thing I could think would be to wrap the Resolver in a "New Type" which could capture the config. sent to it an expose that. That way, we could:

a. better test that each branch of get_resolver results in the right DNS resolvers being referenced (and should mitigate the need to actually make DNS requests in the tests).
b. potentially derive a StructOpt from_os_str for it and create it upstream in Opts, isolating a bit more from the parsing code.

…but that seemed a bit much after a hefty rebase 😁

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm aware in PyTest that you can add "tags" to tests, so you could create tests like "pytest --tag online" to test the online features

that way we can keep it to testing offline for locally and online for CI etc....

Not sure if cargo supports that, and again it isn't really a good solution :(

Jeeze, testing networking tools is hard!

}
11 changes: 10 additions & 1 deletion src/input.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ pub struct Opts {
#[structopt(long)]
pub accessible: bool,

/// A comma-delimited list or file of DNS resolvers.
#[structopt(long)]
pub resolver: Option<String>,

/// The batch size for port scanning, it increases or slows the speed of
/// scanning. Depends on the open file limit of your OS. If you do 65535
/// it will do every port at the same time. Although, your OS may not
Expand Down Expand Up @@ -208,7 +212,7 @@ impl Opts {
self.ports = Some(ports);
}

merge_optional!(range, ulimit, exclude_ports);
merge_optional!(range, resolver, ulimit, exclude_ports);
}
}

Expand All @@ -225,6 +229,7 @@ impl Default for Opts {
ulimit: None,
command: vec![],
accessible: false,
resolver: None,
scan_order: ScanOrder::Serial,
no_config: true,
top: false,
Expand All @@ -250,6 +255,7 @@ pub struct Config {
timeout: Option<u32>,
tries: Option<u8>,
ulimit: Option<u64>,
resolver: Option<String>,
scan_order: Option<ScanOrder>,
command: Option<Vec<String>>,
scripts: Option<ScriptsRequired>,
Expand Down Expand Up @@ -317,6 +323,7 @@ mod tests {
ulimit: None,
command: Some(vec!["-A".to_owned()]),
accessible: Some(true),
resolver: None,
scan_order: Some(ScanOrder::Random),
scripts: None,
exclude_ports: None,
Expand Down Expand Up @@ -364,10 +371,12 @@ mod tests {
end: 1_000,
});
config.ulimit = Some(1_000);
config.resolver = Some("1.1.1.1".to_owned());

opts.merge_optional(&config);

assert_eq!(opts.range, config.range);
assert_eq!(opts.ulimit, config.ulimit);
assert_eq!(opts.resolver, config.resolver);
}
}
Loading