From 63f09328ecaa7893f85c3b9e462ae36239984ffa Mon Sep 17 00:00:00 2001 From: M09ic Date: Tue, 21 Dec 2021 00:34:49 +0800 Subject: [PATCH 1/5] add zoomeye domain api source --- v2/pkg/passive/sources.go | 4 + v2/pkg/runner/config.go | 4 + .../sources/zoomeyeapi/zoomeyeapi.go | 80 +++++++++++++++++++ v2/pkg/subscraping/types.go | 1 + 4 files changed, 89 insertions(+) create mode 100644 v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 72b9a3e78..933a31295 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -36,6 +36,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeye" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" ) // DefaultSources contains the list of fast sources used by default. @@ -114,6 +115,7 @@ var DefaultAllSources = []string{ "virustotal", "waybackarchive", "zoomeye", + "zoomeyeapi", "fofa", } @@ -205,6 +207,8 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &waybackarchive.Source{} case "zoomeye": a.sources[source] = &zoomeye.Source{} + case "zoomeyeapi": + a.sources[source] = &zoomeyeapi.Source{} case "fofa": a.sources[source] = &fofa.Source{} } diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 0d6d2f9a2..8d15bee4d 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -47,6 +47,7 @@ type ConfigFile struct { URLScan []string `yaml:"urlscan"` Virustotal []string `yaml:"virustotal"` ZoomEye []string `yaml:"zoomeye"` + ZoomEyeApi []string `yaml:"zoomeyeapi"` Fofa []string `yaml:"fofa"` // Version indicates the version of subfinder installed. Version string `yaml:"subfinder-version"` @@ -199,6 +200,9 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { keys.ZoomEyePassword = parts[1] } } + if len(c.ZoomEyeApi) > 0 { + keys.ZoomEyeKey = c.ZoomEyeApi[rand.Intn(len(c.ZoomEyeApi))] + } if len(c.Fofa) > 0 { fofaKeys := c.Fofa[rand.Intn(len(c.Fofa))] parts := strings.Split(fofaKeys, ":") diff --git a/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go new file mode 100644 index 000000000..9ea3a63a6 --- /dev/null +++ b/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go @@ -0,0 +1,80 @@ +package zoomeyeapi + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +//type loginResp struct { +// JWT string `json:"access_token"` +//} + +// search results +type zoomeyeResults struct { + Status int `json:"status"` + Total int `json:"total"` + List []struct { + Name string `json:"name"` + Ip []string `json:"ip"` + } `json:"list"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + if session.Keys.ZoomEyeKey == "" { + return + } + + headers := map[string]string{ + "API-KEY": session.Keys.ZoomEyeKey, + "Accept": "application/json", + "Content-Type": "application/json", + } + var pages = 1 + for currentPage := 1; currentPage <= pages; currentPage++ { + api := fmt.Sprintf("https://api.zoomeye.org/domain/search?q=%s&type=1&s=1000&page=%d", domain, currentPage) + resp, err := session.Get(ctx, api, "", headers) + isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden + if err != nil { + if !isForbidden && currentPage == 0 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + } + return + } + + var res zoomeyeResults + err = json.NewDecoder(resp.Body).Decode(&res) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + pages = int(res.Total/1000) + 1 + for _, r := range res.List { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} + } + currentPage++ + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "zoomeyeapi" +} diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 5d9525214..753d125b3 100644 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -61,6 +61,7 @@ type Keys struct { Virustotal string `json:"virustotal"` ZoomEyeUsername string `json:"zoomeye_username"` ZoomEyePassword string `json:"zoomeye_password"` + ZoomEyeKey string `json:"zoomeye_key"` FofaUsername string `json:"fofa_username"` FofaSecret string `json:"fofa_secret"` } From e0ab487a6413a2cd25f5a4f938a05afa334f68c6 Mon Sep 17 00:00:00 2001 From: M09ic Date: Tue, 21 Dec 2021 00:49:38 +0800 Subject: [PATCH 2/5] fix warning --- v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go index 9ea3a63a6..906c46fef 100644 --- a/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go +++ b/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go @@ -59,15 +59,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = json.NewDecoder(resp.Body).Decode(&res) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() + _ = resp.Body.Close() return } - resp.Body.Close() pages = int(res.Total/1000) + 1 for _, r := range res.List { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} } - currentPage++ } }() From 8dacf7c36b3a6308d802d15aa96d0907371def87 Mon Sep 17 00:00:00 2001 From: M09ic Date: Tue, 21 Dec 2021 00:54:23 +0800 Subject: [PATCH 3/5] close zoomeyeapi resp --- v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go index 906c46fef..f4a67dcff 100644 --- a/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go +++ b/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go @@ -57,11 +57,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var res zoomeyeResults err = json.NewDecoder(resp.Body).Decode(&res) + if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} _ = resp.Body.Close() return } + _ = resp.Body.Close() pages = int(res.Total/1000) + 1 for _, r := range res.List { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} From 341575cce20c9a41a8ca7ffd6c834c7306d55794 Mon Sep 17 00:00:00 2001 From: M09ic Date: Tue, 21 Dec 2021 01:04:46 +0800 Subject: [PATCH 4/5] fix zoomeye turn page bug --- v2/pkg/subscraping/sources/zoomeye/zoomeye.go | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/zoomeye/zoomeye.go b/v2/pkg/subscraping/sources/zoomeye/zoomeye.go index a06db86a7..8a093742c 100644 --- a/v2/pkg/subscraping/sources/zoomeye/zoomeye.go +++ b/v2/pkg/subscraping/sources/zoomeye/zoomeye.go @@ -87,7 +87,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain} } } - currentPage++ } }() From 7118f42f15283275192a98a6d0783b2da64bf0e6 Mon Sep 17 00:00:00 2001 From: M09ic Date: Wed, 22 Dec 2021 16:40:38 +0800 Subject: [PATCH 5/5] remove useless comment fix error output --- v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go index f4a67dcff..53f8f6960 100644 --- a/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go +++ b/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go @@ -9,10 +9,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) -//type loginResp struct { -// JWT string `json:"access_token"` -//} - // search results type zoomeyeResults struct { Status int `json:"status"` @@ -48,7 +44,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se resp, err := session.Get(ctx, api, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden if err != nil { - if !isForbidden && currentPage == 0 { + if !isForbidden { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} session.DiscardHTTPResponse(resp) }