-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* new source dnsrepo * added api keys for dnsrepo * source added to test case * integration test set up and test case for dnsrepo * lint error fix * misc update Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
- Loading branch information
1 parent
09d58ad
commit b6ed45c
Showing
11 changed files
with
286 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
| "strings" | ||
|
|
||
| "github.com/logrusorgru/aurora" | ||
|
|
||
| "github.com/projectdiscovery/subfinder/v2/pkg/testutils" | ||
| ) | ||
|
|
||
| var ( | ||
| debug = os.Getenv("DEBUG") == "true" | ||
| githubAction = os.Getenv("GH_ACTION") == "true" | ||
| customTests = os.Getenv("TESTS") | ||
|
|
||
| success = aurora.Green("[✓]").String() | ||
| failed = aurora.Red("[✘]").String() | ||
|
|
||
| sourceTests = map[string]testutils.TestCase{ | ||
| "dnsrepo": dnsrepoTestcases{}, | ||
| } | ||
| ) | ||
|
|
||
| func main() { | ||
| failedTestCases := runTests(toMap(toSlice(customTests))) | ||
|
|
||
| if len(failedTestCases) > 0 { | ||
| if githubAction { | ||
| debug = true | ||
| fmt.Println("::group::Failed integration tests in debug mode") | ||
| _ = runTests(failedTestCases) | ||
| fmt.Println("::endgroup::") | ||
| } | ||
| os.Exit(1) | ||
| } | ||
| } | ||
|
|
||
| func runTests(customTestCases map[string]struct{}) map[string]struct{} { | ||
| failedTestCases := map[string]struct{}{} | ||
|
|
||
| for source, testCase := range sourceTests { | ||
| if len(customTestCases) == 0 { | ||
| fmt.Printf("Running test cases for %q source\n", aurora.Blue(source)) | ||
| } | ||
| if err, failedTemplatePath := execute(source, testCase); err != nil { | ||
| failedTestCases[failedTemplatePath] = struct{}{} | ||
| } | ||
| } | ||
| return failedTestCases | ||
| } | ||
|
|
||
| func execute(source string, testCase testutils.TestCase) (error, string) { | ||
| if err := testCase.Execute(); err != nil { | ||
| _, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, source, err) | ||
| return err, source | ||
| } | ||
|
|
||
| fmt.Printf("%s Test \"%s\" passed!\n", success, source) | ||
| return nil, "" | ||
| } | ||
|
|
||
| func expectResultsGreaterThanCount(results []string, expectedNumber int) error { | ||
| if len(results) > expectedNumber { | ||
| return nil | ||
| } | ||
| return fmt.Errorf("incorrect number of results: expected a result greater than %d,but got %d", expectedNumber, len(results)) | ||
| } | ||
| func toSlice(value string) []string { | ||
| if strings.TrimSpace(value) == "" { | ||
| return []string{} | ||
| } | ||
|
|
||
| return strings.Split(value, ",") | ||
| } | ||
|
|
||
| func toMap(slice []string) map[string]struct{} { | ||
| result := make(map[string]struct{}, len(slice)) | ||
| for _, value := range slice { | ||
| if _, ok := result[value]; !ok { | ||
| result[value] = struct{}{} | ||
| } | ||
| } | ||
| return result | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| #!/bin/bash | ||
|
|
||
| echo "::task~> Clean up & Build binaries files" | ||
| rm integration-test subfinder 2>/dev/null | ||
| cd ../subfinder | ||
| go build | ||
| mv subfinder ../integration-test/subfinder | ||
| cd ../integration-test | ||
| go build | ||
| echo "::done::" | ||
| echo "::task~> Run integration test" | ||
| ./integration-test | ||
| echo "::done::" | ||
| if [ $? -eq 0 ] | ||
| then | ||
| exit 0 | ||
| else | ||
| exit 1 | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "os" | ||
|
|
||
| "github.com/projectdiscovery/subfinder/v2/pkg/testutils" | ||
| ) | ||
|
|
||
| type dnsrepoTestcases struct{} | ||
|
|
||
| func (h dnsrepoTestcases) Execute() error { | ||
| token := os.Getenv("DNSREPO_API_KEY") | ||
| if token == "" { | ||
| return errors.New("missing dns repo api key") | ||
| } | ||
| dnsToken := fmt.Sprintf(`dnsrepo: [%s]`, token) | ||
| file, err := os.CreateTemp("", "provider.yaml") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer os.RemoveAll(file.Name()) | ||
| _, err = file.WriteString(dnsToken) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| results, err := testutils.RunSubfinderAndGetResults(debug, "hackerone.com", "-s", "dnsrepo", "-provider-config", file.Name()) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return expectResultsGreaterThanCount(results, 0) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| package dnsrepo | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "io/ioutil" | ||
| "strings" | ||
|
|
||
| "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | ||
| ) | ||
|
|
||
| // Source is the passive scraping agent | ||
| type Source struct { | ||
| apiKeys []string | ||
| } | ||
|
|
||
| type DnsRepoResponse []struct { | ||
| Domain string | ||
| } | ||
|
|
||
| func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | ||
| results := make(chan subscraping.Result) | ||
|
|
||
| go func() { | ||
| defer close(results) | ||
|
|
||
| randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) | ||
| if randomApiKey == "" { | ||
| return | ||
| } | ||
| resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://dnsrepo.noc.org/api/?apikey=%s&search=%s", randomApiKey, domain)) | ||
| if err != nil { | ||
| results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | ||
| session.DiscardHTTPResponse(resp) | ||
| return | ||
| } | ||
| responseData, err := ioutil.ReadAll(resp.Body) | ||
| if err != nil { | ||
| results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | ||
| session.DiscardHTTPResponse(resp) | ||
| return | ||
| } | ||
| resp.Body.Close() | ||
| var result DnsRepoResponse | ||
| err = json.Unmarshal(responseData, &result) | ||
| if err != nil { | ||
| results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | ||
| session.DiscardHTTPResponse(resp) | ||
| return | ||
| } | ||
| for _, sub := range result { | ||
| results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, ".")} | ||
| } | ||
|
|
||
| }() | ||
| return results | ||
| } | ||
|
|
||
| // Name returns the name of the source | ||
| func (s *Source) Name() string { | ||
| return "dnsrepo" | ||
| } | ||
|
|
||
| func (s *Source) IsDefault() bool { | ||
| return true | ||
| } | ||
|
|
||
| func (s *Source) HasRecursiveSupport() bool { | ||
| return false | ||
| } | ||
|
|
||
| func (s *Source) NeedsKey() bool { | ||
| return true | ||
| } | ||
|
|
||
| func (s *Source) AddApiKeys(keys []string) { | ||
| s.apiKeys = keys | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package testutils | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
| "os/exec" | ||
| "strings" | ||
| ) | ||
|
|
||
| func RunSubfinderAndGetResults(debug bool, domain string, extra ...string) ([]string, error) { | ||
| cmd := exec.Command("bash", "-c") | ||
| cmdLine := fmt.Sprintf("echo %s | %s", domain, "./subfinder ") | ||
| cmdLine += strings.Join(extra, " ") | ||
| cmd.Args = append(cmd.Args, cmdLine) | ||
| if debug { | ||
| cmd.Args = append(cmd.Args, "-v") | ||
| cmd.Stderr = os.Stderr | ||
| fmt.Println(cmd.String()) | ||
| } else { | ||
| cmd.Args = append(cmd.Args, "-silent") | ||
| } | ||
| data, err := cmd.Output() | ||
| if debug { | ||
| fmt.Println(string(data)) | ||
| } | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| var parts []string | ||
| items := strings.Split(string(data), "\n") | ||
| for _, i := range items { | ||
| if i != "" { | ||
| parts = append(parts, i) | ||
| } | ||
| } | ||
| return parts, nil | ||
| } | ||
|
|
||
| // TestCase is a single integration test case | ||
| type TestCase interface { | ||
| Execute() error | ||
| } |