fix eHoleFinger、localFinger指纹默认为and，fg指纹默认为or关系 2022-07-15 17:11:1657…

…876297

main.go

@@ -4,14 +4,15 @@ import (
 	"embed"
 	"fmt"
 	"github.com/hktalent/scan4all/pkg"
-	"github.com/hktalent/scan4all/pkg/fingerprint"
 	naaburunner "github.com/hktalent/scan4all/pkg/naabu/v2/pkg/runner"
+	"github.com/hktalent/scan4all/pocs_go"
 	"github.com/projectdiscovery/gologger"
 	"io"
 	"log"
 	"net/http"
 	_ "net/http/pprof"
 	"runtime"
+	"sync"
 )
 
 //go:embed config/*
@@ -22,14 +23,18 @@ func init() {
 }
 
 func main() {
+	var Close = make(chan bool)
+	var wg sync.WaitGroup
+	go pocs_go.DoNmapScan(Close, &wg)
 	defer func() {
 		log.Println("start close cache, StopCPUProfile... ")
 		pkg.Cache1.Close()
 		//if "true" == pkg.GetVal("autoRmCache") {
 		//	os.RemoveAll(pkg.GetVal(pkg.CacheName))
 		//}
 		// clear
-		fingerprint.ClearData()
+		// 程序都结束了，没有必要清理内存了
+		// fingerprint.ClearData()
 	}()
 	options := naaburunner.ParseOptions()
 	if options.Debug {
@@ -63,4 +68,6 @@ func main() {
 	if err != nil {
 		gologger.Fatal().Msgf("naabuRunner.Httpxrun Could not run httpRunner: %s\n", err)
 	}
+	wg.Wait()
+	close(Close)
 }

pkg/fingerprint/fgConst.go

@@ -105,7 +105,7 @@ func MergeReqUrl() {
 			if x6, ok := oFingerprint[szKey]; ok {
 				x2 = x6
 			} else {
-				x2 = &Fingerprint{Cms: szName, UrlPath: Get4K(&y, "url"), Keyword: []string{}, Id: id, Method: FgType[idMethod], Location: FgType[idPart]}
+				x2 = &Fingerprint{Cms: szName, KeywordMathOr: true, UrlPath: Get4K(&y, "url"), Keyword: []string{}, Id: id, Method: FgType[idMethod], Location: FgType[idPart]}
 				//x1.Fingerprint = append([]Fingerprint{x2}, x1.Fingerprint...)
 				x1.Fingerprint = append(x1.Fingerprint, x2)
 				//log.Println(szKey)

pkg/fingerprint/fingerScan.go

@@ -102,38 +102,38 @@ func CaseMethod(szUrl, method, bodyString, favhash, md5Body, hexBody string, fin
 
 	switch method {
 	case "keyword":
-		if ok, rMz := iskeyword(bodyString, finp.Keyword); ok {
+		if ok, rMz := iskeyword(bodyString, finp.Keyword, finp.KeywordMathOr); ok {
 			cms = append(cms, finp.Cms)
 			SvUrl2Id(szUrl, finp, rMz)
 		}
 		break
 	case "faviconhash": // 相同目标只执行一次
-		if ok, rMz := iskeyword(favhash, finp.Keyword); ok {
+		if ok, rMz := iskeyword(favhash, finp.Keyword, finp.KeywordMathOr); ok {
 			Mfavhash.Store(u01.Host+favhash, 1)
 			cms = append(cms, finp.Cms)
 			SvUrl2Id(szUrl, finp, rMz)
 		}
 		break
 	case "regular":
-		if ok, rMz := isregular(bodyString, finp.Keyword); ok {
+		if ok, rMz := isregular(bodyString, finp.Keyword, finp.KeywordMathOr); ok {
 			cms = append(cms, finp.Cms)
 			SvUrl2Id(szUrl, finp, rMz)
 		}
 		break
 	case "md5": // 支持md5
-		if ok, rMz := iskeyword(md5Body, finp.Keyword); ok {
+		if ok, rMz := iskeyword(md5Body, finp.Keyword, finp.KeywordMathOr); ok {
 			cms = append(cms, finp.Cms)
 			SvUrl2Id(szUrl, finp, rMz)
 		}
 		break
 	case "base64": // 支持base64
-		if ok, rMz := iskeyword(bodyString, finp.Keyword); ok {
+		if ok, rMz := iskeyword(bodyString, finp.Keyword, finp.KeywordMathOr); ok {
 			cms = append(cms, finp.Cms)
 			SvUrl2Id(szUrl, finp, rMz)
 		}
 		break
 	case "hex":
-		if ok, rMz := iskeyword(hexBody, finp.Keyword); ok {
+		if ok, rMz := iskeyword(hexBody, finp.Keyword, finp.KeywordMathOr); ok {
 			cms = append(cms, finp.Cms)
 			SvUrl2Id(szUrl, finp, rMz)
 		}
@@ -187,7 +187,7 @@ func FingerScan(headers map[string][]string, body []byte, title string, url stri
 				} else if finp.Location == "title" { // 识别区域： title
 					cms = append(cms, CaseMethod(url, finp.Method, title, favhash, md5Title, hexTitle, finp)...)
 				} else if finp.Location == "status_code" { // 识别区域：status_code
-					if ok, rMz := iskeyword(status_code, finp.Keyword); ok {
+					if ok, rMz := iskeyword(status_code, finp.Keyword, finp.KeywordMathOr); ok {
 						cms = append(cms, finp.Cms)
 						SvUrl2Id(url, finp, rMz)
 					}

pkg/fingerprint/loadFinger.go

@@ -9,12 +9,13 @@ type Packjson struct {
 }
 
 type Fingerprint struct {
-	Cms      string
-	Method   string
-	Location string
-	Keyword  []string
-	Id       int    // 扩展id属性，通过id关联到组件
-	UrlPath  string // 扩展，有的指纹必须是和特定path关联，例如状态码
+	Cms           string
+	Method        string
+	Location      string
+	Keyword       []string
+	KeywordMathOr bool   // Keyword是否为or关系
+	Id            int    // 扩展id属性，通过id关联到组件
+	UrlPath       string // 扩展，有的指纹必须是和特定path关联，例如状态码
 }
 
 var (

pkg/fingerprint/matchfinger.go

@@ -6,20 +6,24 @@ import (
 	"strings"
 )
 
-func iskeyword(str string, keyword []string) (x bool, rstr string) {
+func iskeyword(str string, keyword []string, KeywordMathOr bool) (x bool, rstr string) {
+	x = true
 	for _, k := range keyword {
 		if strings.Contains(strings.ToLower(str), strings.ToLower(k)) {
-			x = true
+			x = x && true
 			rstr = k
-			break
+			if KeywordMathOr {
+				break
+			}
 		} else {
-			x = false
+			x = x && false
 		}
 	}
 	return x, rstr
 }
 
-func isregular(str string, keyword []string) (x bool, rstr string) {
+func isregular(str string, keyword []string, KeywordMathOr bool) (x bool, rstr string) {
+	x = true
 	for _, k := range keyword {
 		re, err := regexp.Compile(k)
 		if nil != err {
@@ -29,11 +33,13 @@ func isregular(str string, keyword []string) (x bool, rstr string) {
 		//re := pcre.MustCompile(k, pcre.DOTALL)
 		if re.Match([]byte(str)) {
 			//if re.MatcherString(str, pcre.DOTALL).Matches() {
-			x = true
+			x = x && true
 			rstr = k
-			break
+			if KeywordMathOr {
+				break
+			}
 		} else {
-			x = false
+			x = x && false
 		}
 	}
 	return x, rstr

pkg/httpx/runner/runner.go

@@ -1300,7 +1300,7 @@ retry:
 				return nn
 			}
 			//通过wFingerprint获取到的指纹进行检测gopoc check
-			poctechnologies1 = pocs_go.POCcheck(technologies, ul, finalURL, false)
+			poctechnologies1 = pocs_go.POCcheck(technologies, ul, finalURL, false, nil)
 			Vullist = append(Vullist, poctechnologies1...)
 			for _, technology := range technologies {
 				pocYmlList1 := pocs_yml.Check(ul, scanopts.CeyeApi, scanopts.CeyeDomain, r.options.HTTPProxy, strings.ToLower(technology)) // 通过wFingerprint获取到的指纹进行ymlpoc check
@@ -1312,7 +1312,7 @@ retry:
 			// 取差集合
 			filefuzzTechnologies = difference(filefuzzTechnologies, technologies)
 
-			poctechnologies2 = pocs_go.POCcheck(filefuzzTechnologies, ul, finalURL, true) //通过敏感文件扫描获取到的指纹进行检测gopoc check
+			poctechnologies2 = pocs_go.POCcheck(filefuzzTechnologies, ul, finalURL, true, nil) //通过敏感文件扫描获取到的指纹进行检测gopoc check
 			Vullist = append(Vullist, poctechnologies2...)
 			for _, technology := range filefuzzTechnologies {
 				pocYmlList2 := pocs_yml.Check(ul, scanopts.CeyeApi, scanopts.CeyeDomain, r.options.HTTPProxy, strings.ToLower(technology)) //通过敏感文件扫描获取到的指纹进行检测ymlpoc check

pkg/hydra/doNmapResult.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/antchfx/xmlquery"
 	"github.com/hktalent/scan4all/pkg"
+	"github.com/hktalent/scan4all/pocs_go"
 	"io/ioutil"
 	"log"
 	"os"
@@ -49,7 +50,8 @@ func DoParseXml(s string, bf *bytes.Buffer) {
 				port, _ := strconv.Atoi(szPort)
 				service := GetAttr(x.SelectElement("service").Attr, "name")
 				//bf.Write([]byte(fmt.Sprintf("%s:%s\n", ip, szPort)))
-				bf.Write([]byte(fmt.Sprintf("http://%s:%s\n", ip, szPort)))
+				szUlr := fmt.Sprintf("http://%s:%s\n", ip, szPort)
+				bf.Write([]byte(szUlr))
 				go CheckWeakPassword(ip, service, port)
 				// 存储结果到其他地方
 				//x9 := AuthInfo{IPAddr: ip, Port: port, Protocol: service}
@@ -59,6 +61,12 @@ func DoParseXml(s string, bf *bytes.Buffer) {
 						xx09 = a1
 					}
 					m1[ip] = append(xx09, []string{szPort, service})
+					if "445" == szPort && service == "microsoft-ds" {
+						pocs_go.Apc <- &pocs_go.AsyncPocCheck{
+							Wappalyzertechnologies: []string{service},
+							URL:                    szUlr,
+						}
+					}
 				}
 				fmt.Printf("%s\t%d\t%s\n", ip, port, service)
 			}

pocs_go/go_poc_check.go

@@ -13,6 +13,7 @@ import (
 	"github.com/hktalent/scan4all/pocs_go/jboss"
 	"github.com/hktalent/scan4all/pocs_go/jenkins"
 	"github.com/hktalent/scan4all/pocs_go/log4j"
+	"github.com/hktalent/scan4all/pocs_go/ms"
 	"github.com/hktalent/scan4all/pocs_go/phpunit"
 	"github.com/hktalent/scan4all/pocs_go/seeyon"
 	"github.com/hktalent/scan4all/pocs_go/shiro"
@@ -22,19 +23,53 @@ import (
 	"github.com/hktalent/scan4all/pocs_go/zabbix"
 	"log"
 	"net/url"
+	"sync"
 )
 
-func POCcheck(wappalyzertechnologies []string, URL string, finalURL string, checklog4j bool) []string {
-	var HOST string
+type AsyncPocCheck struct {
+	Wappalyzertechnologies []string
+	URL                    string
+	FinalURL               string
+	Checklog4j             bool
+}
+
+var Apc = make(chan *AsyncPocCheck, 300)
+
+func DoNmapScan(close chan bool, wg *sync.WaitGroup) {
+	for {
+		select {
+		case <-close:
+			return
+		case x1 := <-Apc:
+			if nil != x1 {
+				wg.Add(1)
+				go POCcheck(x1.Wappalyzertechnologies, x1.URL, x1.FinalURL, x1.Checklog4j, wg)
+			}
+		}
+	}
+}
+
+// 需优化：相同都目标，相同都检测只做一次
+func POCcheck(wappalyzertechnologies []string, URL string, finalURL string, checklog4j bool, wg *sync.WaitGroup) []string {
+	if nil != wg {
+		defer wg.Done()
+	}
+	var HOST, hostname string
 	var technologies []string
 	if host, err := url.Parse(URL); err == nil {
 		HOST = host.Host
+		hostname = host.Hostname()
 	} else {
 		log.Println(URL, " parse error ", err)
 		return []string{}
 	}
 	for tech := range wappalyzertechnologies {
 		switch wappalyzertechnologies[tech] {
+		case "microsoft-ds":
+			key, err := ms.SmbGhostScan(hostname)
+			if nil == err && key {
+				technologies = append(technologies, fmt.Sprintf("exp-microsoft-ds CVE-2020-0796 :%s", hostname))
+			}
 		case "Shiro":
 			key := shiro.CVE_2016_4437(finalURL)
 			if key != "" {