From 1188bfbc79abe53258d8f69e6bfccd21d858546c Mon Sep 17 00:00:00 2001 From: r0nk Date: Wed, 19 Oct 2022 16:44:42 -0500 Subject: [PATCH 01/19] Prevent EnumerateSubdomains() from continuing if there is an error in Regex (#672) * Prevent EnumerateSubdomains() from continuing if there is an error in Regex * closing with defer Co-authored-by: Sandeep Singh Co-authored-by: r0nk Co-authored-by: mzack --- v2/pkg/passive/passive.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index 7de3fd80c..71b27e7f6 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -14,9 +14,12 @@ import ( func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { + defer close(results) + session, err := subscraping.NewSession(domain, proxy, rateLimit, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} + return } ctx, cancel := context.WithTimeout(context.Background(), maxEnumTime) @@ -49,9 +52,7 @@ func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit, time gologger.Verbose().Label(source).Msg(data) } - close(results) cancel() }() - return results } From d6347c537fe3761d2bbe8502a0ad7e8f1fc6cd8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 29 Oct 2022 14:56:56 +0200 Subject: [PATCH 02/19] chore(deps): bump github.com/projectdiscovery/fileutil in /v2 (#684) Bumps [github.com/projectdiscovery/fileutil](https://github.com/projectdiscovery/fileutil) from 0.0.1 to 0.0.3. - [Release notes](https://github.com/projectdiscovery/fileutil/releases) - [Commits](https://github.com/projectdiscovery/fileutil/compare/v0.0.1...v0.0.3) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fileutil dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 14 +++++++------- v2/go.sum | 31 +++++++++++++++++-------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index cb093aa06..888c47e4f 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -10,11 +10,11 @@ require ( github.com/projectdiscovery/chaos-client v0.3.0 github.com/projectdiscovery/dnsx v1.1.1 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.1 + github.com/projectdiscovery/fileutil v0.0.3 github.com/projectdiscovery/gologger v1.1.5-0.20220817095646-8663411b1b0b github.com/projectdiscovery/ratelimit v0.0.1 github.com/rs/xid v1.4.0 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e gopkg.in/yaml.v3 v3.0.1 @@ -27,7 +27,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/mholt/archiver v3.1.1+incompatible // indirect - github.com/microcosm-cc/bluemonday v1.0.20 // indirect + github.com/microcosm-cc/bluemonday v1.0.21 // indirect github.com/nwaples/rardecode v1.1.0 // indirect github.com/pierrec/lz4 v2.6.0+incompatible // indirect github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect @@ -44,7 +44,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/text v0.4.0 // indirect golang.org/x/tools v0.1.12 // indirect gopkg.in/djherbis/times.v1 v1.2.0 // indirect ) @@ -60,7 +60,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/goflags v0.1.1 github.com/projectdiscovery/retryabledns v1.0.15 // indirect - github.com/projectdiscovery/stringsutil v0.0.1 // indirect - golang.org/x/net v0.0.0-20220927171203-f486391704dc // indirect - golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect + github.com/projectdiscovery/stringsutil v0.0.2 // indirect + golang.org/x/net v0.1.0 // indirect + golang.org/x/sys v0.1.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 5a37b9617..5a084210e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -52,8 +52,8 @@ github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1w github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= -github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y= -github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= +github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= +github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -88,8 +88,8 @@ github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIre github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= -github.com/projectdiscovery/fileutil v0.0.1 h1:3K3UqCDOan3LsvWhV0nyvVuMWSwCloNPUJIGcXsi1os= -github.com/projectdiscovery/fileutil v0.0.1/go.mod h1:Oo6ZEvXmQz/xPF0YukzmwpdW2LYinWCSEmzZOQsJCLg= +github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg= +github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= github.com/projectdiscovery/goflags v0.1.1 h1:AEtT14D9OC10HWyZwDQaSLjuK8ZKoBrSYlsLItvMKZI= github.com/projectdiscovery/goflags v0.1.1/go.mod h1:/YBPA+1igSkQbwD7a91o0HUIwMDlsmQDRZL2oSYSyEQ= @@ -106,8 +106,6 @@ github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8/go.mod h1: github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v1.0.1 h1:eaLBRrImwlYXv8vbXTwR4sxoQqIxR3Y5k/Sd7HhTIII= github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw= -github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa h1:XTRcow+zyZUYxOlLsSp1AdM4Jl+i/OpPbOZPsfU/IAI= -github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa/go.mod h1:zenrIElIcKg0Y9h7pMfTlw5vaI/kCl8uxXm+PfgbBSw= github.com/projectdiscovery/ratelimit v0.0.1 h1:GnCfbKmkLdDLXT3QS4KS0zCsuDGkoRQE0YDbTqzQmS8= github.com/projectdiscovery/ratelimit v0.0.1/go.mod h1:zenrIElIcKg0Y9h7pMfTlw5vaI/kCl8uxXm+PfgbBSw= github.com/projectdiscovery/retryabledns v1.0.15 h1:3Nn119UwYsfUPC3g0q57ftz0Wb5Zl5ppvw8R0Xu0DEI= @@ -121,14 +119,15 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= -github.com/projectdiscovery/stringsutil v0.0.1 h1:a6TCMT+D1aUsoZxNiYf9O30wiDOoLOHDwj89HBjr5BQ= -github.com/projectdiscovery/stringsutil v0.0.1/go.mod h1:TDi2LEqR3OML0BxGoMbbfAHSk5AdfHX762Oc302sgmM= +github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= +github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -136,8 +135,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.7.3/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= @@ -174,9 +174,9 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20220927171203-f486391704dc h1:FxpXZdoBqT8RjqTy6i1E8nXHhW21wK7ptQ/EPIGxzPQ= -golang.org/x/net v0.0.0-20220927171203-f486391704dc/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -195,15 +195,18 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80= golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= From 51cf3677a87892718367393cf4b7766597a3abe5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 29 Oct 2022 14:57:06 +0200 Subject: [PATCH 03/19] chore(deps): bump github.com/stretchr/testify from 1.8.0 to 1.8.1 in /v2 (#683) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.0 to 1.8.1. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.0...v1.8.1) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> From c6eca3ba5b0d3f469325fb97ec1badfba5b8e375 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 29 Oct 2022 14:57:26 +0200 Subject: [PATCH 04/19] chore(deps): bump golangci/golangci-lint-action from 3.2.0 to 3.3.0 (#681) Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.2.0 to 3.3.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v3.2.0...v3.3.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 03d59af42..fa1ee9832 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -14,7 +14,7 @@ jobs: with: go-version: 1.18 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.2.0 + uses: golangci/golangci-lint-action@v3.3.0 with: version: latest args: --timeout 5m From 819583c0ebbc6682252b3fdffb4098bc73b515d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor?= Date: Tue, 1 Nov 2022 11:38:12 +0100 Subject: [PATCH 05/19] Dynamic years range starting from the current year and going back five years. (#687) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Víctor Zamanillo --- .../sources/commoncrawl/commoncrawl.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go index b3073a2c0..09a16f30a 100644 --- a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ b/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -6,14 +6,21 @@ import ( "context" "fmt" "net/url" + "strconv" "strings" + "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) -const indexURL = "https://index.commoncrawl.org/collinfo.json" +const ( + indexURL = "https://index.commoncrawl.org/collinfo.json" + maxYearsBack = 5 +) + +var year = time.Now().Year() type indexResponse struct { ID string `json:"id"` @@ -23,8 +30,6 @@ type indexResponse struct { // Source is the passive scraping agent type Source struct{} -var years = [...]string{"2020", "2019", "2018", "2017"} - // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) @@ -48,6 +53,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } resp.Body.Close() + years := make([]string, 0) + for i := 0; i < maxYearsBack; i++ { + years = append(years, strconv.Itoa(year-i)) + } + searchIndexes := make(map[string]string) for _, year := range years { for _, index := range indexes { From e57de1cf1d576ac32e1f4ed0160304aeae4361f4 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 4 Nov 2022 14:36:44 +0200 Subject: [PATCH 06/19] documentation: update readme with using subfinder as a library #691 --- README.md | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 04263bda5..4060a72de 100644 --- a/README.md +++ b/README.md @@ -236,30 +236,24 @@ package main import ( "bytes" - "context" "fmt" "io" "log" - "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" "github.com/projectdiscovery/subfinder/v2/pkg/runner" ) func main() { runnerInstance, err := runner.NewRunner(&runner.Options{ - Threads: 10, // Thread controls the number of threads to use for active enumerations - Timeout: 30, // Timeout is the seconds to wait for sources to respond - MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration + Threads: 10, // Thread controls the number of threads to use for active enumerations + Timeout: 30, // Timeout is the seconds to wait for sources to respond + MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration Resolvers: resolve.DefaultResolvers, // Use the default list of resolvers by marshaling it to the config - Sources: passive.DefaultSources, // Use the default list of passive sources - AllSources: passive.DefaultAllSources, // Use the default list of all passive sources - Recursive: passive.DefaultRecursiveSources, // Use the default list of recursive sources - Providers: &runner.Providers{}, // Use empty api keys for all providers - }) + }) buf := bytes.Buffer{} - err = runnerInstance.EnumerateSingleDomain(context.Background(), "projectdiscovery.io", []io.Writer{&buf}) + err = runnerInstance.EnumerateSingleDomain("projectdiscovery.io", []io.Writer{&buf}) if err != nil { log.Fatal(err) } From e289b74737a2536e42ccc7164c93f9b07d12b6b1 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 5 Nov 2022 10:03:04 +0100 Subject: [PATCH 07/19] adding recon.cloud source (#679) --- README.md | 2 +- v2/pkg/passive/sources.go | 2 + v2/pkg/passive/sources_test.go | 7 +- .../sources/reconcloud/reconcloud.go | 81 +++++++++++++++++++ 4 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 v2/pkg/subscraping/sources/reconcloud/reconcloud.go diff --git a/README.md b/README.md index 04263bda5..224934f82 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key: -[BeVigil](https://bevigil.com/osint-api), [Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [Zoomeye](https://www.zoomeye.org) +[BeVigil](https://bevigil.com/osint-api), [Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [Zoomeye](https://www.zoomeye.org), [Recon.Cloud](https://recon.cloud/) These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration. diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 2544742f0..5076b5ab9 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -31,6 +31,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/passivetotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/reconcloud" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/riddler" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" @@ -83,6 +84,7 @@ var AllSources = [...]subscraping.Source{ &zoomeye.Source{}, &zoomeyeapi.Source{}, &dnsrepo.Source{}, + &reconcloud.Source{}, } var NameSourceMap = make(map[string]subscraping.Source, len(AllSources)) diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 4ff093fc6..8b47409e6 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -47,6 +47,7 @@ var ( "whoisxmlapi", "zoomeye", "zoomeyeapi", + "reconcloud", } expectedDefaultSources = []string{ @@ -75,6 +76,7 @@ var ( "threatminer", "virustotal", "whoisxmlapi", + "reconcloud", } expectedDefaultRecursiveSources = []string{ @@ -89,6 +91,7 @@ var ( "securitytrails", "sonarsearch", "virustotal", + "reconcloud", } ) @@ -134,13 +137,13 @@ func TestSourceFiltering(t *testing.T) { {someSources, someExclusions, false, false, len(someSources) - len(someExclusions)}, {someSources, someExclusions, false, true, 1}, {someSources, someExclusions, true, false, len(AllSources) - len(someExclusions)}, - {someSources, someExclusions, true, true, 9}, + {someSources, someExclusions, true, true, 10}, {someSources, []string{}, false, false, len(someSources)}, {someSources, []string{}, true, false, len(AllSources)}, {[]string{}, []string{}, false, false, len(expectedDefaultSources)}, - {[]string{}, []string{}, false, true, 9}, + {[]string{}, []string{}, false, true, 10}, {[]string{}, []string{}, true, false, len(AllSources)}, {[]string{}, []string{}, true, true, len(expectedDefaultRecursiveSources)}, } diff --git a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go b/v2/pkg/subscraping/sources/reconcloud/reconcloud.go new file mode 100644 index 000000000..670ebfdad --- /dev/null +++ b/v2/pkg/subscraping/sources/reconcloud/reconcloud.go @@ -0,0 +1,81 @@ +// Package reconcloud logic +package reconcloud + +import ( + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type reconCloudResponse struct { + MsgType string `json:"msg_type"` + RequestID string `json:"request_id"` + OnCache bool `json:"on_cache"` + Step string `json:"step"` + CloudAssetsList []cloudAssetsList `json:"cloud_assets_list"` +} + +type cloudAssetsList struct { + Key string `json:"key"` + Domain string `json:"domain"` + CloudProvider string `json:"cloud_provider"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain)) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response reconCloudResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if len(response.CloudAssetsList) > 0 { + for _, cloudAsset := range response.CloudAssetsList { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "reconcloud" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} From cd39e9e3ef805199b8b78de3dd7b5187a0eec5c6 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 5 Nov 2022 10:34:20 +0100 Subject: [PATCH 08/19] adding hunter source (#680) Co-authored-by: Sandeep Singh --- README.md | 2 +- v2/pkg/passive/sources.go | 3 + v2/pkg/passive/sources_test.go | 2 + v2/pkg/subscraping/sources/hunter/hunter.go | 105 ++++++++++++++++++++ 4 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 v2/pkg/subscraping/sources/hunter/hunter.go diff --git a/README.md b/README.md index 224934f82..45732eaab 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key: -[BeVigil](https://bevigil.com/osint-api), [Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [Zoomeye](https://www.zoomeye.org), [Recon.Cloud](https://recon.cloud/) +[BeVigil](https://bevigil.com/osint-api), [Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [Zoomeye](https://www.zoomeye.org), [Hunter](https://hunter.qianxin.com/), [Recon.Cloud](https://recon.cloud/) These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration. diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 5076b5ab9..6c139f0ed 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -27,6 +27,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/passivetotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" @@ -84,7 +85,9 @@ var AllSources = [...]subscraping.Source{ &zoomeye.Source{}, &zoomeyeapi.Source{}, &dnsrepo.Source{}, + &hunter.Source{}, &reconcloud.Source{}, + } var NameSourceMap = make(map[string]subscraping.Source, len(AllSources)) diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 8b47409e6..6bedebd32 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -47,6 +47,7 @@ var ( "whoisxmlapi", "zoomeye", "zoomeyeapi", + "hunter", "reconcloud", } @@ -76,6 +77,7 @@ var ( "threatminer", "virustotal", "whoisxmlapi", + "hunter", "reconcloud", } diff --git a/v2/pkg/subscraping/sources/hunter/hunter.go b/v2/pkg/subscraping/sources/hunter/hunter.go new file mode 100644 index 000000000..ff960e27f --- /dev/null +++ b/v2/pkg/subscraping/sources/hunter/hunter.go @@ -0,0 +1,105 @@ +package hunter + +import ( + "context" + "encoding/base64" + "fmt" + + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type hunterResp struct { + Code int `json:"code"` + Data hunterData `json:"data"` + Message string `json:"message"` +} + +type infoArr struct { + URL string `json:"url"` + IP string `json:"ip"` + Port int `json:"port"` + Domain string `json:"domain"` + Protocol string `json:"protocol"` +} + +type hunterData struct { + InfoArr []infoArr `json:"arr"` + Total int `json:"total"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + var pages = 1 + for currentPage := 1; currentPage <= pages; currentPage++ { + // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2 + qbase64 := base64.URLEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=1&page_size=100&is_web=3", randomApiKey, qbase64)) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response hunterResp + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if response.Code == 401 || response.Code == 400 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message)} + return + } + + if response.Data.Total > 0 { + for _, hunterInfo := range response.Data.InfoArr { + subdomain := hunterInfo.Domain + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + pages = int(response.Data.Total/1000) + 1 + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "hunter" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} From 84e33c925f0c1e582b0c3d8f15bb640e9b247c2d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Nov 2022 15:05:24 +0530 Subject: [PATCH 09/19] chore(deps): bump github.com/projectdiscovery/goflags in /v2 (#690) Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.1 to 0.1.3. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.1...v0.1.3) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 888c47e4f..0033a7b33 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -58,7 +58,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/goflags v0.1.1 + github.com/projectdiscovery/goflags v0.1.3 github.com/projectdiscovery/retryabledns v1.0.15 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect golang.org/x/net v0.1.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 5a084210e..0a7374c79 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -91,8 +91,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg= github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= -github.com/projectdiscovery/goflags v0.1.1 h1:AEtT14D9OC10HWyZwDQaSLjuK8ZKoBrSYlsLItvMKZI= -github.com/projectdiscovery/goflags v0.1.1/go.mod h1:/YBPA+1igSkQbwD7a91o0HUIwMDlsmQDRZL2oSYSyEQ= +github.com/projectdiscovery/goflags v0.1.3 h1:dnJlg19VkDp1iYkpAod4Tv+OAngr7Mq61LMMpBQlO0M= +github.com/projectdiscovery/goflags v0.1.3/go.mod h1:/7ZAoY1SVfUcGobTP5QDvGQmrpPDDlBUDIMr7c+r94Q= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/gologger v1.1.5-0.20220817095646-8663411b1b0b h1:sncWNStu8+oT3vDvKKFncr5FxEui5Bs0ET2Qkj0AVBo= From 7da3b7864dd31b44f06afcb23ec20955cad54ba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor?= Date: Wed, 9 Nov 2022 10:31:11 +0100 Subject: [PATCH 10/19] Fix crtsh email values (#676) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fixes #675, crsth can return email values for specific domains so we always try to extract the subdomain using the default regular expression of the session * using session for uniformity with http signature Co-authored-by: Víctor Zamanillo Co-authored-by: Mzack9999 --- v2/pkg/subscraping/sources/crtsh/crtsh.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/v2/pkg/subscraping/sources/crtsh/crtsh.go index 93c1f895b..41f8904e1 100644 --- a/v2/pkg/subscraping/sources/crtsh/crtsh.go +++ b/v2/pkg/subscraping/sources/crtsh/crtsh.go @@ -30,7 +30,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se go func() { defer close(results) - count := s.getSubdomainsFromSQL(domain, results) + count := s.getSubdomainsFromSQL(domain, session, results) if count > 0 { return } @@ -40,7 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return results } -func (s *Source) getSubdomainsFromSQL(domain string, results chan subscraping.Result) int { +func (s *Source) getSubdomainsFromSQL(domain string, session *subscraping.Session, results chan subscraping.Result) int { db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes") if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -73,7 +73,7 @@ func (s *Source) getSubdomainsFromSQL(domain string, results chan subscraping.Re return count } count++ - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data} + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: session.Extractor.FindString(data)} } return count } @@ -98,7 +98,7 @@ func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, sessi for _, subdomain := range subdomains { for _, sub := range strings.Split(subdomain.NameValue, "\n") { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: sub} + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: session.Extractor.FindString(sub)} } } From acefa5cb8c9bd470fbb9c08cf789a83231ff6dff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Nov 2022 15:02:44 +0530 Subject: [PATCH 11/19] chore(deps): bump golang from 1.19.2-alpine to 1.19.3-alpine (#697) Bumps golang from 1.19.2-alpine to 1.19.3-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 617b77168..b0268b7f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.19.2-alpine AS build-env +FROM golang:1.19.3-alpine AS build-env RUN go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest # Release From c681154806c64783b6c25bc7ca483a55f19ece30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor?= Date: Wed, 9 Nov 2022 12:24:35 +0100 Subject: [PATCH 12/19] omnisint from Sonar Labs is dead, is not working anymore (#701) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Víctor Zamanillo --- DISCLAIMER.md | 2 +- v2/pkg/passive/sources.go | 2 - v2/pkg/passive/sources_test.go | 3 - .../sources/sonarsearch/sonarsearch.go | 74 ------------------- 4 files changed, 1 insertion(+), 80 deletions(-) delete mode 100644 v2/pkg/subscraping/sources/sonarsearch/sonarsearch.go diff --git a/DISCLAIMER.md b/DISCLAIMER.md index 54da2a428..114a152ec 100644 --- a/DISCLAIMER.md +++ b/DISCLAIMER.md @@ -2,7 +2,7 @@ Subfinder leverages multiple open APIs, it is developed for individuals to help them for research or internal work. If you wish to incorporate this tool into a commercial offering or purposes, you must agree to the Terms of the leveraged services: -- Project Sonar / Bufferover: https://opendata.rapid7.com/about / [https://tls.bufferover.run](https://tls.bufferover.run/dns?q) +- Bufferover: https://tls.bufferover.run - CommonCrawl: https://commoncrawl.org/terms-of-use/full - certspotter: https://sslmate.com/terms - dnsdumpster: https://hackertarget.com/terms diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 6c139f0ed..127107783 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -38,7 +38,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sonarsearch" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatminer" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" @@ -76,7 +75,6 @@ var AllSources = [...]subscraping.Source{ &securitytrails.Source{}, &shodan.Source{}, &sitedossier.Source{}, - &sonarsearch.Source{}, &threatbook.Source{}, &threatminer.Source{}, &virustotal.Source{}, diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 6bedebd32..04c28ef75 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -39,7 +39,6 @@ var ( "securitytrails", "shodan", "sitedossier", - "sonarsearch", "threatbook", "threatminer", "virustotal", @@ -91,7 +90,6 @@ var ( "hackertarget", "passivetotal", "securitytrails", - "sonarsearch", "virustotal", "reconcloud", } @@ -119,7 +117,6 @@ func TestSourceCategorization(t *testing.T) { func TestSourceFiltering(t *testing.T) { someSources := []string{ "alienvault", - "sonarsearch", "chaos", "virustotal", } diff --git a/v2/pkg/subscraping/sources/sonarsearch/sonarsearch.go b/v2/pkg/subscraping/sources/sonarsearch/sonarsearch.go deleted file mode 100644 index 3b9c389bc..000000000 --- a/v2/pkg/subscraping/sources/sonarsearch/sonarsearch.go +++ /dev/null @@ -1,74 +0,0 @@ -// Package sonarsearch logic -package sonarsearch - -import ( - "context" - "encoding/json" - "fmt" - "strconv" - - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - go func() { - defer close(results) - - getURL := fmt.Sprintf("https://sonar.omnisint.io/subdomains/%s?page=", domain) - page := 0 - var subdomains []string - for { - resp, err := session.SimpleGet(ctx, getURL+strconv.Itoa(page)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHTTPResponse(resp) - return - } - - if err := json.NewDecoder(resp.Body).Decode(&subdomains); err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return - } - resp.Body.Close() - - if len(subdomains) == 0 { - return - } - - for _, subdomain := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - - page++ - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "sonarsearch" -} - -func (s *Source) IsDefault() bool { - return false -} - -func (s *Source) HasRecursiveSupport() bool { - return true -} - -func (s *Source) NeedsKey() bool { - return false -} - -func (s *Source) AddApiKeys(_ []string) { - // no key needed -} From ab54ff4f35afd29fbfd7d32ec7eecc625fbeaa95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor?= Date: Wed, 9 Nov 2022 12:38:41 +0100 Subject: [PATCH 13/19] Fixed source tests, added crtsh with recursion (#702) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Víctor Zamanillo --- v2/pkg/passive/sources_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 04c28ef75..367be7c66 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -118,6 +118,7 @@ func TestSourceFiltering(t *testing.T) { someSources := []string{ "alienvault", "chaos", + "crtsh", "virustotal", } @@ -136,7 +137,7 @@ func TestSourceFiltering(t *testing.T) { {someSources, someExclusions, false, false, len(someSources) - len(someExclusions)}, {someSources, someExclusions, false, true, 1}, {someSources, someExclusions, true, false, len(AllSources) - len(someExclusions)}, - {someSources, someExclusions, true, true, 10}, + {someSources, someExclusions, true, true, 9}, {someSources, []string{}, false, false, len(someSources)}, {someSources, []string{}, true, false, len(AllSources)}, From 86e1d1a916b4773135dc57ac3b936d971630fba7 Mon Sep 17 00:00:00 2001 From: Shubham Rasal Date: Sat, 12 Nov 2022 00:10:15 +0530 Subject: [PATCH 14/19] Add callback function (#699) * Add callback function * Rename OnResult to ResultCallback * Add callback documentation * Update README.md --- README.md | 21 +++++++++------------ v2/pkg/runner/enumerate.go | 5 +++++ v2/pkg/runner/options.go | 4 ++++ 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 45732eaab..afc818336 100644 --- a/README.md +++ b/README.md @@ -236,30 +236,27 @@ package main import ( "bytes" - "context" "fmt" "io" "log" - "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" "github.com/projectdiscovery/subfinder/v2/pkg/runner" ) func main() { - runnerInstance, err := runner.NewRunner(&runner.Options{ - Threads: 10, // Thread controls the number of threads to use for active enumerations - Timeout: 30, // Timeout is the seconds to wait for sources to respond - MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration + runnerInstance, _ := runner.NewRunner(&runner.Options{ + Threads: 10, // Thread controls the number of threads to use for active enumerations + Timeout: 30, // Timeout is the seconds to wait for sources to respond + MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration Resolvers: resolve.DefaultResolvers, // Use the default list of resolvers by marshaling it to the config - Sources: passive.DefaultSources, // Use the default list of passive sources - AllSources: passive.DefaultAllSources, // Use the default list of all passive sources - Recursive: passive.DefaultRecursiveSources, // Use the default list of recursive sources - Providers: &runner.Providers{}, // Use empty api keys for all providers - }) + ResultCallback: func(s *resolve.HostEntry) { // Callback function to execute for available host + log.Println(s.Host, s.Source) + }, + }) buf := bytes.Buffer{} - err = runnerInstance.EnumerateSingleDomain(context.Background(), "projectdiscovery.io", []io.Writer{&buf}) + err := runnerInstance.EnumerateSingleDomain("projectdiscovery.io", []io.Writer{&buf}) if err != nil { log.Fatal(err) } diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index a352f0a70..9c15c707f 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -141,6 +141,11 @@ func (r *Runner) EnumerateSingleDomain(domain string, writers []io.Writer) error numberOfSubDomains = len(uniqueMap) } + if r.options.ResultCallback != nil { + for _, v := range uniqueMap { + r.options.ResultCallback(&v) + } + } gologger.Info().Msgf("Found %d subdomains for '%s' in %s\n", numberOfSubDomains, domain, duration) return nil diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 0cec7419d..e38df4162 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -62,8 +62,12 @@ type Options struct { Filter goflags.StringSlice matchRegexes []*regexp.Regexp filterRegexes []*regexp.Regexp + ResultCallback OnResultCallback // OnResult callback } +// OnResultCallback (hostResult) +type OnResultCallback func(result *resolve.HostEntry) + // ParseOptions parses the command line flags provided by a user func ParseOptions() *Options { // Seed default random number generator From 84cb9536989b12e0c88ec8b2ce32614df0c1dbcd Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 17 Nov 2022 19:43:59 +0530 Subject: [PATCH 15/19] version update --- v2/pkg/runner/banners.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index b2e7bfb67..991a1efef 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -9,11 +9,11 @@ const banner = ` _______ __/ /_ / __(_)___ ____/ /__ _____ / ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/ (__ ) /_/ / /_/ / __/ / / / / /_/ / __/ / -/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.5.4 +/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.5.5 ` // Version is the current version of subfinder -const Version = `v2.5.4` +const Version = `v2.5.5` // showBanner is used to show the banner to the user func showBanner() { From 6150f4c560a9549dcb93346209e6f8c073b4f328 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Nov 2022 19:46:59 +0530 Subject: [PATCH 16/19] chore(deps): bump github.com/projectdiscovery/fdmax in /v2 (#710) Bumps [github.com/projectdiscovery/fdmax](https://github.com/projectdiscovery/fdmax) from 0.0.3 to 0.0.4. - [Release notes](https://github.com/projectdiscovery/fdmax/releases) - [Commits](https://github.com/projectdiscovery/fdmax/compare/v0.0.3...v0.0.4) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fdmax dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 4 ++-- v2/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 0033a7b33..f20b5c5fb 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -9,7 +9,7 @@ require ( github.com/lib/pq v1.10.7 github.com/projectdiscovery/chaos-client v0.3.0 github.com/projectdiscovery/dnsx v1.1.1 - github.com/projectdiscovery/fdmax v0.0.3 + github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/fileutil v0.0.3 github.com/projectdiscovery/gologger v1.1.5-0.20220817095646-8663411b1b0b github.com/projectdiscovery/ratelimit v0.0.1 @@ -62,5 +62,5 @@ require ( github.com/projectdiscovery/retryabledns v1.0.15 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect golang.org/x/net v0.1.0 // indirect - golang.org/x/sys v0.1.0 // indirect + golang.org/x/sys v0.2.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 0a7374c79..fface0d16 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -83,8 +83,8 @@ github.com/projectdiscovery/chaos-client v0.3.0 h1:A4NgOYRCrlsSZUBTCT2HAT/uTEJly github.com/projectdiscovery/chaos-client v0.3.0/go.mod h1:AWx/KZgtBE5SULpsgyQLLfb+SQgVtCih83gvRtTpwl4= github.com/projectdiscovery/dnsx v1.1.1 h1:yGYEH1vfVN7YdvdOtPzPvrc6JuHCi8wBTAkStP/f2QI= github.com/projectdiscovery/dnsx v1.1.1/go.mod h1:DVvc+ePRCknahLpz4Y8nMppYOGUZhkEmYsTPuYx1a5w= -github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= -github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= +github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= +github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= @@ -186,7 +186,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -196,8 +195,9 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 929ae28e9a4e37906d705f1c921836a60dc2e169 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Nov 2022 19:47:32 +0530 Subject: [PATCH 17/19] chore(deps): bump golangci/golangci-lint-action from 3.3.0 to 3.3.1 (#705) Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.3.0 to 3.3.1. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v3.3.0...v3.3.1) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index fa1ee9832..86aaaafff 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -14,7 +14,7 @@ jobs: with: go-version: 1.18 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.3.0 + uses: golangci/golangci-lint-action@v3.3.1 with: version: latest args: --timeout 5m From e0048f01e46f22a8ad04639bdafbc41e66ddd793 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Nov 2022 19:53:39 +0530 Subject: [PATCH 18/19] chore(deps): bump alpine from 3.16.2 to 3.16.3 (#704) Bumps alpine from 3.16.2 to 3.16.3. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 98b634429..bbec2b6ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ RUN apk add build-base RUN go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest # Release -FROM alpine:3.16.2 +FROM alpine:3.16.3 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=build-env /go/bin/subfinder /usr/local/bin/subfinder From 16ecbfb286442ca9a516be591b4c072815a44f06 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 17 Nov 2022 16:25:16 +0200 Subject: [PATCH 19/19] Readme update (#707) * go version update * list of sources requiring an API key * etc --- README.md | 119 +++++++++++++++++++++++++++++------------------------- 1 file changed, 64 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index afc818336..9e9cff115 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- subfinder + subfinder

@@ -13,7 +13,7 @@

- +

Features • Install • @@ -26,10 +26,11 @@ --- -Subfinder is a subdomain discovery tool that discovers valid subdomains for websites by using passive online sources. It has a simple modular architecture and is optimized for speed. subfinder is built for doing one thing only - passive subdomain enumeration, and it does that very well. - -We have designed `subfinder` to comply with all passive sources licenses, and usage restrictions, as well as maintained a consistently passive model to make it useful to both penetration testers and bug bounty hunters alike. +`subfinder` is a subdomain discovery tool that returns valid subdomains for websites, using passive online sources. It has a simple, modular architecture and is optimized for speed. `subfinder` is built for +doing one thing only - passive subdomain enumeration, and it does that very well. +We have made it to comply with all the used passive source licenses and usage restrictions. The passive model guarantees speed and stealthiness that can be leveraged by both penetration testers and bug bounty +hunters alike. # Features @@ -38,89 +39,93 @@ We have designed `subfinder` to comply with all passive sources licenses, and us
- - - Fast and powerful resolution and wildcard elimination module - - **Curated** passive sources to maximize results - - Multiple Output formats supported (Json, File, Stdout) - - Optimized for speed, very fast and **lightweight** on resources - - **STDIN/OUT** support for integrating in workflows - +- Fast and powerful resolution and wildcard elimination modules +- **Curated** passive sources to maximize results +- Multiple output formats supported (JSON, file, stdout) +- Optimized for speed and **lightweight** on resources +- **STDIN/OUT** support enables easy integration into workflows # Usage ```sh subfinder -h ``` + This will display help for the tool. Here are all the switches it supports. ```yaml +Usage: + ./subfinder [flags] + Flags: INPUT: - -d, -domain string[] domains to find subdomains for - -dL, -list string file containing list of domains for subdomain discovery + -d, -domain string[] domains to find subdomains for + -dL, -list string file containing list of domains for subdomain discovery SOURCE: - -s, -sources string[] specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources. - -recursive use only sources that can handle subdomains recursively (e.g. subdomain.domain.tld vs domain.tld) - -all use all sources for enumeration (slow) - -es, -exclude-sources string[] sources to exclude from enumeration (-es alienvault,zoomeye) + -s, -sources string[] specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources. + -recursive use only sources that can handle subdomains recursively (e.g. subdomain.domain.tld vs domain.tld) + -all use all sources for enumeration (slow) + -es, -exclude-sources string[] sources to exclude from enumeration (-es alienvault,zoomeye) FILTER: - -m, -match string[] subdomain or list of subdomain to match (file or comma separated) - -f, -filter string[] subdomain or list of subdomain to filter (file or comma separated) + -m, -match string[] subdomain or list of subdomain to match (file or comma separated) + -f, -filter string[] subdomain or list of subdomain to filter (file or comma separated) RATE-LIMIT: - -rl, -rate-limit int maximum number of http requests to send per second - -t int number of concurrent goroutines for resolving (-active only) (default 10) + -rl, -rate-limit int maximum number of http requests to send per second + -t int number of concurrent goroutines for resolving (-active only) (default 10) OUTPUT: - -o, -output string file to write output to - -oJ, -json write output in JSONL(ines) format - -oD, -output-dir string directory to write output (-dL only) - -cs, -collect-sources include all sources in the output (-json only) - -oI, -ip include host IP in output (-active only) + -o, -output string file to write output to + -oJ, -json write output in JSONL(ines) format + -oD, -output-dir string directory to write output (-dL only) + -cs, -collect-sources include all sources in the output (-json only) + -oI, -ip include host IP in output (-active only) CONFIGURATION: - -config string flag config file (default "$HOME/.config/subfinder/config.yaml") - -pc, -provider-config string provider config file (default "$HOME/.config/subfinder/provider-config.yaml") - -r string[] comma separated list of resolvers to use - -rL, -rlist string file containing list of resolvers to use - -nW, -active display active subdomains only - -proxy string http proxy to use with subfinder - -ei, -exclude-ip exclude IPs from the list of domains + -config string flag config file (default "$HOME/.config/subfinder/config.yaml") + -pc, -provider-config string provider config file (default "$HOME/.config/subfinder/provider-config.yaml") + -r string[] comma separated list of resolvers to use + -rL, -rlist string file containing list of resolvers to use + -nW, -active display active subdomains only + -proxy string http proxy to use with subfinder + -ei, -exclude-ip exclude IPs from the list of domains DEBUG: - -silent show only subdomains in output - -version show version of subfinder - -v show verbose output - -nc, -no-color disable color in output - -ls, -list-sources list all available sources + -silent show only subdomains in output + -version show version of subfinder + -v show verbose output + -nc, -no-color disable color in output + -ls, -list-sources list all available sources OPTIMIZATION: - -timeout int seconds to wait before timing out (default 30) - -max-time int minutes to wait for enumeration results (default 10) + -timeout int seconds to wait before timing out (default 30) + -max-time int minutes to wait for enumeration results (default 10) ``` # Installation -Subfinder requires **go1.17** to install successfully. Run the following command to install the latest version: +`subfinder` requires **go1.18** to install successfully. Run the following command to install the latest version: ```sh go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest ``` - ## Post Installation Instructions -Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key: +`subfinder` can be used right after the installation, however the following services require configuring API keys to work: + +[BeVigil](https://bevigil.com/osint-api), [BinaryEdge](https://binaryedge.io), [BufferOver](https://tls.bufferover.run), [C99](https://api.c99.nl/), [Censys](https://censys.io), [CertSpotter](https://sslmate.com/certspotter/api/), [Chaos](https://chaos.projectdiscovery.io), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [FullHunt](https://fullhunt.io), [GitHub](https://github.com), [Intelx](https://intelx.io), [PassiveTotal](http://passivetotal.org), [quake](https://quake.360.cn), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [ThreatBook](https://x.threatbook.cn/en), [VirusTotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [ZoomEye](https://www.zoomeye.org), [ZoomEye API](https://api.zoomeye.org), [dnsrepo](https://dnsrepo.noc.org), [Hunter](https://hunter.qianxin.com/) -[BeVigil](https://bevigil.com/osint-api), [Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [Zoomeye](https://www.zoomeye.org), [Hunter](https://hunter.qianxin.com/), [Recon.Cloud](https://recon.cloud/) +You can also use the `subfinder -ls` command to display all the available sources. -These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration. +These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys +can be specified for each of these services from which one of them will be used for enumeration. -For sources that require multiple keys, namely `Censys`, `Passivetotal`, they can be added by separating them via a colon (:). +Composite keys for sources like, `Censys`, `PassiveTotal`, `Fofa`, `Intellix` and `ZoomEye`, need to be separated with a colon (`:`). -An example provider config file - +An example provider config file: ```yaml binaryedge: @@ -137,6 +142,8 @@ shodan: github: - ghp_lkyJGU3jv1xmwk4SDXavrLDJ4dl2pSJMzj4X - ghp_gkUuhkIYdQPj13ifH4KA3cXRn8JD2lqir2d4 +zoomeye: + - zoomeye_username:zoomeye_password ``` # Running Subfinder @@ -182,7 +189,8 @@ events.hackerone.com [INF] Found 18 subdomains for hackerone.com in 3 seconds 672 milliseconds ``` -The subdomains discovered can be piped to other tools too. For example, you can pipe the subdomains discovered by subfinder to httpx [httpx](https://github.com/projectdiscovery/httpx) which will then find running http servers on the host. +The subdomains discovered can be piped to other tools too. For example, you can pipe the discovered subdomains to [`httpx`](https://github.com/projectdiscovery/httpx) which will then find +running HTTP servers on the host. ```console echo hackerone.com | subfinder -silent | httpx -silent @@ -207,13 +215,13 @@ Pull the latest tagged [subfinder](https://hub.docker.com/r/projectdiscovery/sub docker pull projectdiscovery/subfinder:latest ``` -Running subfinder using docker image: +Running `subfinder` using the docker image: ```sh docker run projectdiscovery/subfinder:latest -d hackerone.com ``` -Running subfinder using docker image with local config file: +Running `subfinder` using the docker image, with a local config file: ```sh docker run -v $HOME/.config/subfinder:/root/.config/subfinder -t projectdiscovery/subfinder -d hackerone.com @@ -245,7 +253,7 @@ import ( ) func main() { - runnerInstance, _ := runner.NewRunner(&runner.Options{ + runnerInstance, err := runner.NewRunner(&runner.Options{ Threads: 10, // Thread controls the number of threads to use for active enumerations Timeout: 30, // Timeout is the seconds to wait for sources to respond MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration @@ -256,7 +264,7 @@ func main() { }) buf := bytes.Buffer{} - err := runnerInstance.EnumerateSingleDomain("projectdiscovery.io", []io.Writer{&buf}) + err = runnerInstance.EnumerateSingleDomain("projectdiscovery.io", []io.Writer{&buf}) if err != nil { log.Fatal(err) } @@ -280,6 +288,7 @@ func main() { # License -`subfinder` is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/subfinder/blob/master/THANKS.md)** file for more details. +`subfinder` is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See +the **[THANKS.md](https://github.com/projectdiscovery/subfinder/blob/master/THANKS.md)** file for more details. -Read the disclaimer for usage at [DISCLAIMER.md](https://github.com/projectdiscovery/subfinder/blob/master/DISCLAIMER.md) and [contact us](mailto:contact@projectdiscovery.io) for any API removal. +Read the usage disclaimer at [DISCLAIMER.md](https://github.com/projectdiscovery/subfinder/blob/master/DISCLAIMER.md) and [contact us](mailto:contact@projectdiscovery.io) for any API removal.