goflags support

README.md

@@ -53,34 +53,45 @@ subfinder -h
 ```
 This will display help for the tool. Here are all the switches it supports.
 
-| Flag             | Description                                                | Example                                     |
-| ---------------- | ---------------------------------------------------------- | --------------------------------------------|
-| -all             | Use all sources (slow) for enumeration                     | subfinder -d uber.com -all                  |
-| -b               | IP address to be used as local bind                        | subfinder -b 172.16.0.1                |
-| -config          | Configuration file for API Keys, etc                       | subfinder -config config.yaml               |
-| -d               | Domain to find subdomains for                              | subfinder -d uber.com                       |
-| -dL              | File containing list of domains to enumerate               | subfinder -dL hackerone-hosts.txt           |
-| -exclude-sources | List of sources to exclude from enumeration                | subfinder -exclude-sources archiveis        |
-| -max-time        | Minutes to wait for enumeration results (default 10)       | subfinder -max-time 1                       |
-| -nC              | Don't Use colors in output                                 | subfinder -nC                               |
-| -nW              | Remove Wildcard & Dead Subdomains from output              | subfinder -nW                               |
-| -ls              | List all available sources                                 | subfinder -ls                               |
-| -o               | File to write output to (optional)                         | subfinder -o output.txt                     |
-| -oD              | Directory to write enumeration results to (optional)       | subfinder -oD ~/outputs                     |
-| -oI              | Write output in Host,IP format                             | subfinder -oI                               |
-| -oJ              | Write output in JSON lines Format                          | subfinder -oJ                               |
-| -r               | Comma-separated list of resolvers to use                   | subfinder -r 1.1.1.1,1.0.0.1                |
-| -rL              | Text file containing list of resolvers to use              | subfinder -rL resolvers.txt                 |
-| -recursive       | Enumeration recursive subdomains                           | subfinder -d news.yahoo.com -recursive      |
-| -silent          | Show only subdomains in output                             | subfinder -silent                           |
-| -sources         | Comma separated list of sources to use                     | subfinder -sources shodan,censys            |
-| -t               | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100                            |
-| -timeout         | Seconds to wait before timing out (default 30)             | subfinder -timeout 30                       |
-| -proxy           | HTTP proxy to use with subfinder                           | subfinder -proxy http://localhost:3128      |
-| -rate-limit      | Maximum number of HTTP requests to send per second         | subfinder -rate-limit 10                    |
-| -v               | Show Verbose output                                        | subfinder -v                                |
-| -version         | Show current program version                               | subfinder -version                          |
-
+Flags:
+INPUT:
+   -d, -domain string  Domain to find subdomains for
+   -dL, -list string   File containing list of domains to enumerate
+
+SOURCE:
+   -s, -sources string           Sources to use for enumeration (-s crtsh,bufferover)
+   -recursive                    Sources to use supports recursive enumeration
+   -all                          Use all sources (slow) for enumeration
+   -es, -exclude-sources string  Sources to exclude from enumeration (-es archiveis,zoomeye)
+
+RATE-LIMIT:
+   -rate-limit int  Maximum number of HTTP requests to send per second
+   -t int           Number of concurrent goroutines for resolving (-active only) (default 10)
+
+OUTPUT:
+   -o, -output string       File to write output to (optional)
+   -oJ, -json               Write output in JSONL(ines) format
+   -oD, -output-dir string  Directory to write output (-dL only)
+   -cs, -collect-sources    Include all sources in the output (-json only)
+   -oI, -ip                 Include host IP in output (-active only)
+
+CONFIGURATION:
+   -config string       Configuration file for API Keys, etc
+   -r string            Comma separated list of resolvers to use
+   -rL, -rlist string   File containing list of resolvers to use
+   -nW, -active         Display active subdomains only
+   -proxy string        HTTP proxy to use with subfinder
+
+DEBUG:
+   -silent   Show only subdomains in output
+   -version  Show version of subfinder
+   -v        Show Verbose output
+   -nC, -nc  Disable color in output
+   -ls       List all available sources
+
+OPTIMIZATION:
+   -timeout int   Seconds to wait before timing out (default 30)
+   -max-time int  Minutes to wait for enumeration results (default 10)
 
 # Installation
 

v2/go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/projectdiscovery/chaos-client v0.1.8
 	github.com/projectdiscovery/dnsx v1.0.3
 	github.com/projectdiscovery/fdmax v0.0.3
+	github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/rs/xid v1.3.0
 	github.com/spyse-com/go-spyse v1.2.3
@@ -19,18 +20,24 @@ require (
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
 
+require github.com/karrick/godirwalk v1.16.1 // indirect
+
 require (
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
+	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
 	github.com/miekg/dns v1.1.41 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/projectdiscovery/goflags v0.0.8-0.20220208063718-9bbeacc2fb8f
 	github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e // indirect
+	github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe // indirect
 	golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d // indirect
 	golang.org/x/sys v0.0.0-20210419170143-37df388d1f33 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

v2/go.sum

@@ -7,6 +7,8 @@ github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9or
 github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
+github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
 github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
 github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
 github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
@@ -44,6 +46,8 @@ github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd/go.mod h1:VzxiSdG6j1p
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
+github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -89,12 +93,18 @@ github.com/projectdiscovery/dnsx v1.0.3 h1:IbMlMEeki6mlllHxATVh50+eOBwyOn8OKD1Cl
 github.com/projectdiscovery/dnsx v1.0.3/go.mod h1:M79ADVHwnNQM6kN2J4/XvjydJ3cDhaHHoI5lxJR2Z/A=
 github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg=
 github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI=
+github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA=
+github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
+github.com/projectdiscovery/goflags v0.0.8-0.20220208063718-9bbeacc2fb8f h1:FKTkdM1pPIL0gQRRQDoWjd/mZz+4DZ2Bk1l+ZbOJmIQ=
+github.com/projectdiscovery/goflags v0.0.8-0.20220208063718-9bbeacc2fb8f/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
 github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
 github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e h1:cxke2L/GKym765W0UnA9RuyaY/LI2u5z+fYqDtoQY0M=
 github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
 github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
@@ -207,6 +217,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=

v2/pkg/passive/passive.go

@@ -3,7 +3,6 @@ package passive
 import (
 	"context"
 	"fmt"
-	"net"
 	"sync"
 	"time"
 
@@ -12,11 +11,10 @@ import (
 )
 
 // EnumerateSubdomains enumerates all the subdomains for a given domain
-func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit, timeout int, maxEnumTime time.Duration, localIP net.IP) chan subscraping.Result {
+func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result {
 	results := make(chan subscraping.Result)
-
 	go func() {
-		session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout, localIP)
+		session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout)
 		if err != nil {
 			results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)}
 		}

v2/pkg/passive/sources.go

@@ -17,6 +17,7 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
@@ -38,7 +39,6 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeye"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi"
-	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt"
 )
 
 // DefaultSources contains the list of fast sources used by default.

v2/pkg/runner/banners.go

@@ -27,50 +27,24 @@ func showBanner() {
 	gologger.Print().Msgf("By using subfinder, you also agree to the terms of the APIs used.\n\n")
 }
 
-// normalRunTasks runs the normal startup tasks
-func (options *Options) normalRunTasks() {
-	configFile, err := UnmarshalRead(options.ConfigFile)
-	if err != nil {
-		gologger.Fatal().Msgf("Could not read configuration file %s: %s\n", options.ConfigFile, err)
+// loadProvidersFrom runs the app with source config
+func (options *Options) loadProvidersFrom(location string) {
+	if len(options.AllSources) == 0 {
+		options.AllSources = passive.DefaultAllSources
 	}
-
-	// If we have a different version of subfinder installed
-	// previously, use the new iteration of config file.
-	if configFile.Version != Version {
-		configFile.Sources = passive.DefaultSources
-		configFile.AllSources = passive.DefaultAllSources
-		configFile.Recursive = passive.DefaultRecursiveSources
-		configFile.Version = Version
-
-		err = configFile.MarshalWrite(options.ConfigFile)
-		if err != nil {
-			gologger.Fatal().Msgf("Could not update configuration file to %s: %s\n", options.ConfigFile, err)
-		}
+	if len(options.Recursive) == 0 {
+		options.Recursive = passive.DefaultRecursiveSources
 	}
-	options.YAMLConfig = configFile
-}
-
-// firstRunTasks runs some housekeeping tasks done
-// when the program is ran for the first time
-func (options *Options) firstRunTasks() {
-	// Create the configuration file and display information
-	// about it to the user.
-	config := ConfigFile{
-		// Use the default list of resolvers by marshaling it to the config
-		Resolvers: resolve.DefaultResolvers,
-		// Use the default list of passive sources
-		Sources: passive.DefaultSources,
-		// Use the default list of all passive sources
-		AllSources: passive.DefaultAllSources,
-		// Use the default list of recursive sources
-		Recursive: passive.DefaultRecursiveSources,
+	// todo: move elsewhere
+	if len(options.Resolvers) == 0 {
+		options.Recursive = resolve.DefaultResolvers
 	}
-
-	err := config.MarshalWrite(options.ConfigFile)
-	if err != nil {
-		gologger.Fatal().Msgf("Could not write configuration file to %s: %s\n", options.ConfigFile, err)
+	if len(options.Sources) == 0 {
+		options.Sources = passive.DefaultSources
 	}
-	options.YAMLConfig = config
 
-	gologger.Info().Msgf("Configuration file saved to %s\n", options.ConfigFile)
+	options.Providers = &Providers{}
+	if err := options.Providers.UnmarshalFrom(location); isFatalErr(err) {
+		gologger.Fatal().Msgf("Could not read providers from %s: %s\n", location, err)
+	}
 }

v2/pkg/runner/config.go

@@ -4,7 +4,6 @@ import (
 	"math/rand"
 	"os"
 	"strings"
-	"time"
 
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
 	"gopkg.in/yaml.v3"
@@ -13,21 +12,8 @@ import (
 // MultipleKeyPartsLength is the max length for multiple keys
 const MultipleKeyPartsLength = 2
 
-// YAMLIndentCharLength number of chars for identation on write YAML to file
-const YAMLIndentCharLength = 4
-
-// ConfigFile contains the fields stored in the configuration file
-type ConfigFile struct {
-	// Resolvers contains the list of resolvers to use while resolving
-	Resolvers []string `yaml:"resolvers,omitempty"`
-	// Sources contains a list of sources to use for enumeration
-	Sources []string `yaml:"sources,omitempty"`
-	// AllSources contains the list of all sources for enumeration (slow)
-	AllSources []string `yaml:"all-sources,omitempty"`
-	// Recrusive contains the list of recursive subdomain enum sources
-	Recursive []string `yaml:"recursive,omitempty"`
-	// ExcludeSources contains the sources to not include in the enumeration process
-	ExcludeSources []string `yaml:"exclude-sources,omitempty"`
+// Providers contains the providers stored in the configuration file
+type Providers struct {
 	// API keys for different sources
 	Binaryedge     []string `yaml:"binaryedge"`
 	C99            []string `yaml:"c99"`
@@ -51,15 +37,10 @@ type ConfigFile struct {
 	ZoomEyeApi     []string `yaml:"zoomeyeapi"`
 	Fofa           []string `yaml:"fofa"`
 	FullHunt       []string `json:"fullhunt"`
-	// Version indicates the version of subfinder installed.
-	Version string `yaml:"subfinder-version"`
 }
 
 // GetConfigDirectory gets the subfinder config directory for a user
 func GetConfigDirectory() (string, error) {
-	// Seed the random number generator
-	rand.Seed(time.Now().UnixNano())
-
 	var config string
 
 	directory, err := os.UserHomeDir()
@@ -77,48 +58,32 @@ func GetConfigDirectory() (string, error) {
 	return config, nil
 }
 
-// CheckConfigExists checks if the config file exists in the given path
-func CheckConfigExists(configPath string) bool {
-	if _, err := os.Stat(configPath); err == nil {
-		return true
-	} else if os.IsNotExist(err) {
-		return false
-	}
-	return false
-}
-
-// MarshalWrite writes the marshaled yaml config to disk
-func (c *ConfigFile) MarshalWrite(file string) error {
-	f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
+// MarshalTo writes the marshaled yaml config to disk
+func (c *Providers) MarshalTo(file string) error {
+	f, err := os.Create(file)
 	if err != nil {
 		return err
 	}
+	defer f.Close()
 
-	// Indent the spaces too
-	enc := yaml.NewEncoder(f)
-	enc.SetIndent(YAMLIndentCharLength)
-	err = enc.Encode(&c)
-	f.Close()
-	return err
+	return yaml.NewEncoder(f).Encode(c)
 }
 
-// UnmarshalRead reads the unmarshalled config yaml file from disk
-func UnmarshalRead(file string) (ConfigFile, error) {
-	config := ConfigFile{}
-
+// MarshalTo writes the marshaled yaml config to disk
+func (c *Providers) UnmarshalFrom(file string) error {
 	f, err := os.Open(file)
 	if err != nil {
-		return config, err
+		return err
 	}
-	err = yaml.NewDecoder(f).Decode(&config)
-	f.Close()
-	return config, err
+	defer f.Close()
+
+	return yaml.NewDecoder(f).Decode(c)
 }
 
 // GetKeys gets the API keys from config file and creates a Keys struct
 // We use random selection of api keys from the list of keys supplied.
 // Keys that require 2 options are separated by colon (:).
-func (c *ConfigFile) GetKeys() subscraping.Keys {
+func (c *Providers) GetKeys() subscraping.Keys {
 	keys := subscraping.Keys{}
 
 	if len(c.Binaryedge) > 0 {