Remove unsafe flag

HTTP User-Agent should be randomized by default since subfinder uses third party apis

README.md

@@ -76,7 +76,6 @@ This will display help for the tool. Here are all the switches it supports.
 | -t               | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100                            |
 | -timeout         | Seconds to wait before timing out (default 30)             | subfinder -timeout 30                       |
 | -http-proxy      | Http Proxy                                                 | subfinder -http-proxy http://localhost:3128 |
-| -unsafe          | Send HTTP request without User-Agent header randomization  | subfinder -unsafe                           |
 | -v               | Show Verbose output                                        | subfinder -v                                |
 | -version         | Show current program version                               | subfinder -version                          |
 

v2/pkg/passive/passive.go

@@ -11,11 +11,11 @@ import (
 )
 
 // EnumerateSubdomains enumerates all the subdomains for a given domain
-func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, unsafe bool, timeout int, maxEnumTime time.Duration) chan subscraping.Result {
+func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result {
 	results := make(chan subscraping.Result)
 
 	go func() {
-		session, err := subscraping.NewSession(domain, keys, proxy, unsafe, timeout)
+		session, err := subscraping.NewSession(domain, keys, proxy, timeout)
 		if err != nil {
 			results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)}
 		}

v2/pkg/runner/enumerate.go

@@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
 
 	// Run the passive subdomain enumeration
 	now := time.Now()
-	passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.UnSafe, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute)
+	passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute)
 
 	wg := &sync.WaitGroup{}
 	wg.Add(1)

v2/pkg/runner/options.go

@@ -40,7 +40,6 @@ type Options struct {
 	ResolverList       string     // ResolverList is a text file containing list of resolvers to use for enumeration
 	ConfigFile         string     // ConfigFile contains the location of the config file
 	Proxy              string     // HTTP proxy
-	UnSafe             bool       // Send HTTP request without User-Agent header randomization
 	YAMLConfig         ConfigFile // YAMLConfig contains the unmarshalled yaml config file
 }
 
@@ -78,7 +77,6 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output")
 	flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc")
 	flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use")
-	flag.BoolVar(&options.UnSafe, "unsafe", false, "Send HTTP request without User-Agent header randomization")
 	flag.BoolVar(&options.Version, "version", false, "Show version of subfinder")
 	flag.Parse()
 

v2/pkg/subscraping/agent.go

@@ -15,7 +15,7 @@ import (
 )
 
 // NewSession creates a new session object for a domain
-func NewSession(domain string, keys *Keys, proxy string, unsafe bool, timeout int) (*Session, error) {
+func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) {
 	Transport := &http.Transport{
 		MaxIdleConns:        100,
 		MaxIdleConnsPerHost: 100,
@@ -43,7 +43,6 @@ func NewSession(domain string, keys *Keys, proxy string, unsafe bool, timeout in
 	session := &Session{
 		Client: client,
 		Keys:   keys,
-		UnSafe: unsafe,
 	}
 
 	// Create a new extractor object for the current domain
@@ -80,13 +79,7 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s
 		return nil, err
 	}
 
-	// Unsafe requests do not use user-agent randomization
-	if s.UnSafe {
-		req.Header.Set("User-Agent", "subfinder - Open-source Project (github.com/projectdiscovery/subfinder)")
-	} else {
-		req.Header.Set("User-Agent", uarand.GetRandom())
-	}
-
+	req.Header.Set("User-Agent", uarand.GetRandom())
 	req.Header.Set("Accept", "*/*")
 	req.Header.Set("Accept-Language", "en")
 	req.Header.Set("Connection", "close")

v2/pkg/subscraping/types.go

@@ -31,8 +31,6 @@ type Session struct {
 	Keys *Keys
 	// Client is the current http client
 	Client *http.Client
-	// Perform unsafe HTTP requests without user agent randomization
-	UnSafe bool
 }
 
 // Keys contains the current API Keys we have in store