up 2022-08-23

.github/build/linux.yml

@@ -16,6 +16,7 @@ builds:
       - linux
     goarch:
       - amd64
+      - arm64
 archives:
 - format: zip
 

.github/build/mac.yml

@@ -16,6 +16,7 @@ builds:
       - darwin
     goarch:
       - amd64
+      - arm64
 
 archives:
 - format: zip

.github/build/windows.yml

@@ -3,6 +3,9 @@ env:
 before:
   hooks:
     - go mod tidy
+#after:
+#  hooks:
+#    post: ./upx.sh
 project_name: scan4all
 builds:
   - id: scan4all-windows

.github/workflows/build.yml

@@ -6,26 +6,6 @@ on:
   workflow_dispatch:
 
 jobs:
-  build-mac-arm:
-    runs-on: macos-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.18
-      - name: Install Dependences
-        run: brew install libpcap
-      - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v2
-        with:
-          version: latest
-          args: release -f .github/build/mac_arm64.yml --rm-dist
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   build-mac:
     runs-on: macos-latest
     steps:
@@ -38,7 +18,7 @@ jobs:
         with:
           go-version: 1.18
       - name: Install Dependences
-        run: brew install libpcap
+        run: brew install libpcap upx
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2
         with:
@@ -59,7 +39,7 @@ jobs:
         with:
           go-version: 1.18
       - name: Install Dependences
-        run: sudo apt install -yy libpcap-dev
+        run: sudo apt install -yy libpcap-dev upx
 
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2

.github/workflows/upx.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+for FILE in dist/scan4all_*/*; do
+    du -sh ${FILE}
+    upx ${FILE}
+    du -sh ${FILE}
+done

go.mod

@@ -77,6 +77,7 @@ require github.com/spaolacci/murmur3 v1.1.0
 
 require (
 	github.com/PuerkitoBio/goquery v1.8.0
+	github.com/Qianlitp/crawlergo v0.4.3
 	github.com/RumbleDiscovery/jarm-go v0.0.6
 	github.com/Ullaakut/nmap v2.0.2+incompatible
 	github.com/ammario/ipisp/v2 v2.0.0
@@ -120,6 +121,7 @@ require (
 	github.com/shirou/gopsutil/v3 v3.22.7
 	github.com/sijms/go-ora/v2 v2.4.25
 	github.com/simonnilsson/ask v0.2.0
+	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/viper v1.12.0
 	github.com/stacktitan/smb v0.0.0-20190531122847-da9a425dceb8
 	github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
@@ -160,6 +162,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/deckarep/golang-set v1.7.1 // indirect
 	github.com/dgraph-io/ristretto v0.1.0 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
@@ -178,6 +181,7 @@ require (
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.1.0 // indirect
 	github.com/gofrs/uuid v4.2.0+incompatible // indirect
+	github.com/gogf/gf v1.16.6 // indirect
 	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
 	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
@@ -232,6 +236,7 @@ require (
 	github.com/nwaples/rardecode v1.1.2 // indirect
 	github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 // indirect
 	github.com/owenrumney/go-sarif/v2 v2.1.2 // indirect
+	github.com/panjf2000/ants/v2 v2.2.2 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.1 // indirect

pkg/fingerprint/fgConst.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 	"encoding/json"
 	"fmt"
+	"github.com/hktalent/scan4all/lib/util"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -39,6 +40,8 @@ var FgType map[int]string = map[int]string{
 
 //go:embed dicts/fg.json
 var FgData string
+
+// 指纹  {id：指纹数据对象}
 var FGDataMap []map[string]interface{}
 
 func Get4K(m *map[string]interface{}, k string) string {
@@ -57,11 +60,11 @@ func MergeReqUrl() {
 	LoadWebfingerprintEhole()
 	x1 := GetWebfingerprintEhole()
 	// 测试的时候下面代码才打开
-	//if "true" == pkg.GetValByDefault("MyDebug", "false") {
-	//	x1.Fingerprint = []*Fingerprint{}
-	//	localFinger = "{}"
-	//	log.Println("MyDebug")
-	//}
+	if "true" == util.GetValByDefault("MyDebug", "false") {
+		x1.Fingerprint = []*Fingerprint{}
+		localFinger = "{}"
+		log.Println("MyDebug")
+	}
 
 	// 不重复的URL
 	var urls = []string{}
@@ -134,6 +137,13 @@ func DelTmpFgFile() {
 // 这里可以动态加载远程的url指纹数据到 FgData
 func init() {
 	json.Unmarshal([]byte(FgData), &FGDataMap)
+	var aN []map[string]interface{}
+	for _, x := range FGDataMap {
+		if bD, ok := x["delete"]; ok && false == bD.(bool) {
+			aN = append(aN, x)
+		}
+	}
+	FGDataMap = aN
 	MergeReqUrl()
 	var err error
 	tempInput1, err = ioutil.TempFile("", "dict-in-*")

pkg/fingerprint/fingerScan.go

@@ -3,6 +3,7 @@ package fingerprint
 import (
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"github.com/hktalent/scan4all/lib/util"
 	"log"
 	"net/url"
@@ -50,7 +51,7 @@ var MUrl *sync.Map = new(sync.Map)
 // 图标每个目标只识别一次
 var Mfavhash *sync.Map = new(sync.Map)
 
-//  一个url到底和多少组件id关联
+// 一个url到底和多少组件id关联
 var MFid *sync.Map = new(sync.Map)
 
 // 清除数据
@@ -149,10 +150,10 @@ func CaseMethod(szUrl, method, bodyString, favhash, md5Body, hexBody string, fin
 var enableFingerTitleHeaderMd5Hex = util.GetValAsBool("enableFingerTitleHeaderMd5Hex")
 
 // 相同的url、组件（产品），>=2 个指纹命中，那么该组件的其他指纹匹配将跳过
-func FingerScan(headers map[string][]string, body []byte, title string, url string, status_code string) []string {
+func FingerScan(headers map[string][]string, body []byte, title string, url string, status_code string) ([]string, []string) {
 	if nil == body || 0 == len(body) {
 		//log.Println(url, " 存在异常，body为nil")
-		return []string{}
+		return []string{}, nil
 	}
 	//log.Println("FgDictFile = ", FgDictFile)
 	bodyString := string(body)
@@ -174,8 +175,10 @@ func FingerScan(headers map[string][]string, body []byte, title string, url stri
 	}
 
 	var cms []string
+	var fgIds []string
 	for _, x1 := range []*Packjson{EholeFinpx, LocalFinpx} {
 		for _, finp := range x1.Fingerprint {
+			n1 := len(cms)
 			if finp.UrlPath == "" || strings.HasSuffix(url, finp.UrlPath) {
 				//if -1 < strings.Index(url, "/favicon.ico") && finp.Cms == "SpringBoot" {
 				//	log.Println(url)
@@ -195,7 +198,14 @@ func FingerScan(headers map[string][]string, body []byte, title string, url stri
 					}
 				}
 			}
+			// 找到指纹
+			if len(cms) > n1 {
+				fgIds = append(fgIds, fmt.Sprintf("%v", finp.Id))
+				log.Printf("%d\n", finp.Id)
+				n1 = len(cms)
+			}
 		}
+
 	}
-	return cms
+	return cms, fgIds
 }

pkg/httpx/runner/runner.go

@@ -1258,7 +1258,7 @@ retry:
 		for match := range matches {
 			technologies = append(technologies, match)
 		}
-		matchlocals := fingerprint.FingerScan(resp.Headers, resp.Data, title, ul, strconv.Itoa(resp.StatusCode))
+		matchlocals, _ := fingerprint.FingerScan(resp.Headers, resp.Data, title, ul, strconv.Itoa(resp.StatusCode))
 
 		for _, matchlocal := range matchlocals {
 			technologies = append(technologies, matchlocal)

test/testfg/TestFg2.go

@@ -103,7 +103,7 @@ https://47.104.237.208`, "\n")
 					//log.Println(url1, " error: ", err)
 					return
 				}
-				xx1 := fingerprint.FingerScan(headers, body, title, url2, status_code)
+				xx1, _ := fingerprint.FingerScan(headers, body, title, url2, status_code)
 				if 0 < len(xx1) {
 					log.Printf("%s 指纹 %+v  %s", url1, xx1, status_code)
 				}