Skip to content

Commit

Permalink
Add WhoisXML API support (projectdiscovery#583)
Browse files Browse the repository at this point in the history
  • Loading branch information
VortixDev authored Jul 29, 2022
1 parent 83f6bcb commit 13583c7
Show file tree
Hide file tree
Showing 5 changed files with 81 additions and 1 deletion.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest

Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key:

[Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [Zoomeye](https://www.zoomeye.org)
[Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [Zoomeye](https://www.zoomeye.org)

These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration.

Expand Down
5 changes: 5 additions & 0 deletions v2/pkg/passive/sources.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatminer"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeye"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi"
)
Expand Down Expand Up @@ -64,6 +65,7 @@ var DefaultSources = []string{
"threatcrowd",
"threatminer",
"virustotal",
"whoisxmlapi",
}

// DefaultRecursiveSources contains list of default recursive sources
Expand Down Expand Up @@ -116,6 +118,7 @@ var DefaultAllSources = []string{
"threatminer",
"virustotal",
"waybackarchive",
"whoisxmlapi",
"zoomeye",
"zoomeyeapi",
}
Expand Down Expand Up @@ -204,6 +207,8 @@ func (a *Agent) addSources(sources []string) {
a.sources[source] = &virustotal.Source{}
case "waybackarchive":
a.sources[source] = &waybackarchive.Source{}
case "whoisxmlapi":
a.sources[source] = &whoisxmlapi.Source{}
case "zoomeye":
a.sources[source] = &zoomeye.Source{}
case "zoomeyeapi":
Expand Down
4 changes: 4 additions & 0 deletions v2/pkg/runner/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ type Providers struct {
ThreatBook []string `yaml:"threatbook"`
URLScan []string `yaml:"urlscan"`
Virustotal []string `yaml:"virustotal"`
WhoisXMLAPI []string `yaml:"whoisxmlapi"`
ZoomEye []string `yaml:"zoomeye"`
ZoomEyeApi []string `yaml:"zoomeyeapi"`
}
Expand Down Expand Up @@ -155,6 +156,9 @@ func (c *Providers) GetKeys() subscraping.Keys {
if len(c.Virustotal) > 0 {
keys.Virustotal = sliceutil.PickRandom(c.Virustotal)
}
if len(c.WhoisXMLAPI) > 0 {
keys.WhoisXMLAPI = sliceutil.PickRandom(c.WhoisXMLAPI)
}
if len(c.ZoomEye) > 0 {
zoomEyeKeys := sliceutil.PickRandom(c.ZoomEye)
if keyPartA, keyPartB, ok := multipartKey(zoomEyeKeys); ok {
Expand Down
70 changes: 70 additions & 0 deletions v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
// Package virustotal logic
package whoisxmlapi

import (
"context"
"fmt"

jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)

type response struct {
Search string `json:"search"`
Result Result `json:"result"`
}

type Result struct {
Count int `json:"count"`
Records []Record `json:"records"`
}

type Record struct {
Domain string `json:"domain"`
FirstSeen int `json:"firstSeen"`
LastSeen int `json:"lastSeen"`
}

// Source is the passive scraping agent
type Source struct{}

// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)

go func() {
defer close(results)

if session.Keys.WhoisXMLAPI == "" {
return
}

resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://subdomains.whoisxmlapi.com/api/v1?apiKey=%s&domainName=%s", session.Keys.WhoisXMLAPI, domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
session.DiscardHTTPResponse(resp)
return
}

var data response
err = jsoniter.NewDecoder(resp.Body).Decode(&data)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
resp.Body.Close()
return
}

resp.Body.Close()

for _, record := range data.Result.Records {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}
}
}()

return results
}

// Name returns the name of the source
func (s *Source) Name() string {
return "whoisxmlapi"
}
1 change: 1 addition & 0 deletions v2/pkg/subscraping/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ type Keys struct {
ThreatBook string `json:"threatbook"`
URLScan string `json:"urlscan"`
Virustotal string `json:"virustotal"`
WhoisXMLAPI string `json:"whoisxmlapi"`
ZoomEyeUsername string `json:"zoomeye_username"`
ZoomEyePassword string `json:"zoomeye_password"`
ZoomEyeKey string `json:"zoomeye_key"`
Expand Down

0 comments on commit 13583c7

Please sign in to comment.