Merge pull request #467 from projectdiscovery/issue-449-custom-resolvers

Adding support for custom resolvers

Author: ehsandeep

README.md

@@ -138,6 +138,7 @@ OUTPUT:
    -csv                              Output in CSV format
 
 CONFIGURATIONS:
+   -r, -resolvers string[]       List of custom resolvers (file or comma separated)
    -allow string[]               Allowed list of IP/CIDR's to process (file or comma separated)
    -deny string[]                Denied list of IP/CIDR's to process (file or comma separated)
    -random-agent                 Enable Random User-Agent to use (default true)
@@ -323,7 +324,8 @@ https://support.hackerone.com
 - `vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes.
 - Unique flags should be used for specific use cases instead of running them as default with other flags.
 - When using `json` flag, all the information (default probes) included in the JSON output.
-
+- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port`  (eg **udp:127.0.0.1:53**)
+- Invalid custom resolvers/files are ignored.
 
 # Acknowledgement
 

common/httpx/httpx.go

@@ -44,6 +44,9 @@ func New(options *Options) (*HTTPX, error) {
 	fastdialerOpts.Deny = options.Deny
 	fastdialerOpts.Allow = options.Allow
 	fastdialerOpts.WithDialerHistory = true
+	if len(options.Resolvers) > 0 {
+		fastdialerOpts.BaseResolvers = options.Resolvers
+	}
 	dialer, err := fastdialer.NewDialer(fastdialerOpts)
 	if err != nil {
 		return nil, fmt.Errorf("could not create resolver cache: %s", err)

common/httpx/option.go

@@ -36,6 +36,7 @@ type Options struct {
 	MaxResponseBodySizeToSave int64
 	MaxResponseBodySizeToRead int64
 	UnsafeURI                 string
+	Resolvers                 []string
 }
 
 // DefaultOptions contains the default options

runner/options.go

@@ -4,6 +4,7 @@ import (
 	"math"
 	"os"
 	"regexp"
+	"strings"
 
 	"github.com/projectdiscovery/fileutil"
 	"github.com/projectdiscovery/goconfig"
@@ -197,6 +198,7 @@ type Options struct {
 	Stream                    bool
 	SkipDedupe                bool
 	ProbeAllIPS               bool
+	Resolvers                 goflags.NormalizedStringSlice
 }
 
 // ParseOptions parses the command line options for application
@@ -272,6 +274,7 @@ func ParseOptions() *Options {
 	)
 
 	createGroup(flagSet, "configs", "Configurations",
+		flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "List of custom resolvers (file or comma separated)"),
 		flagSet.Var(&options.Allow, "allow", "Allowed list of IP/CIDR's to process (file or comma separated)"),
 		flagSet.Var(&options.Deny, "deny", "Denied list of IP/CIDR's to process (file or comma separated)"),
 		flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"),
@@ -370,6 +373,26 @@ func (options *Options) validateOptions() {
 			gologger.Fatal().Msgf("Invalid value for match regex option: %s\n", err)
 		}
 	}
+
+	var resolvers []string
+	for _, resolver := range options.Resolvers {
+		if fileutil.FileExists(resolver) {
+			chFile, err := fileutil.ReadFile(resolver)
+			if err != nil {
+				gologger.Fatal().Msgf("Couldn't process resolver file \"%s\": %s\n", resolver, err)
+			}
+			for line := range chFile {
+				resolvers = append(resolvers, line)
+			}
+		} else {
+			resolvers = append(resolvers, resolver)
+		}
+	}
+	options.Resolvers = resolvers
+	if len(options.Resolvers) > 0 {
+		gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ","))
+
+	}
 }
 
 // configureOutput configures the output on the screen

runner/runner.go

@@ -108,6 +108,7 @@ func New(options *Options) (*Runner, error) {
 	if httpxOptions.MaxResponseBodySizeToSave > httpxOptions.MaxResponseBodySizeToRead {
 		httpxOptions.MaxResponseBodySizeToSave = httpxOptions.MaxResponseBodySizeToRead
 	}
+	httpxOptions.Resolvers = options.Resolvers
 
 	var key, value string
 	httpxOptions.CustomHeaders = make(map[string]string)