Skip to content

Commit

Permalink
1、重构了webscan
Browse files Browse the repository at this point in the history 
2、优化filefuzz，重用http1.1 keep-alive
3、优化，优先使用http2.0，提高效率
4、增加veo师傅的poc 2022-10-03
  • Loading branch information
@hktalent
hktalent committed Oct 3, 2022
1 parent 249cd50 commit cc61397
Show file tree
Hide file tree
Showing 44 changed files with 655 additions and 520 deletions.
5 changes: 2 additions & 3 deletions brute/dicts/filedic.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5553,7 +5553,6 @@ admin%20
admin%20/
admin%20/page
admin%24
admin%ext%
admin-ANTIGO
admin-admin
admin-ajax
Expand Down Expand Up @@ -27949,8 +27948,8 @@ url.jsp
url.php
url_1.sql
url_2.sql
urllist.
urllist.txt.gz
urlList.
urlList.txt.gz
urlmem-app/.env
urlogy
urls.py
Expand Down
35 changes: 20 additions & 15 deletions brute/filefuzz.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ func init() {
var RandStr4Cookie = util.RandStringRunes(10)

// 重写了fuzz：优化流程、优化算法、修复线程安全bug、增加智能功能
// 两次 ioutil.ReadAll(resp.Body)，第二次就会 Read返回EOF error
func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody string) ([]string, []string) {
u01, err := url.Parse(strings.TrimSpace(u))
if nil == err {
Expand Down Expand Up @@ -187,22 +188,25 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
var async_technologies = make(chan []string, util.Fuzzthreads*2)
// 字典长度的 70% 的错误
var MaxErrorTimes int32 = int32(float32(len(filedic)) * 0.7)
defer func() {
close(ch)
close(async_data)
close(async_technologies)
}()
//defer func() {
// close(ch)
// close(async_data)
// close(async_technologies)
//}()
//log.Printf("start fuzz: %s for", u)
nStop := 400
go func() {
for {
select {
case <-ctx2.Done():
return
case x1, ok := <-async_data:
if ok {
path = append(path, x1...)
if len(path) > nStop {
stop() //发停止指令
atomic.AddInt32(&errorTimes, MaxErrorTimes)
return
}
} else {
return
Expand All @@ -213,13 +217,12 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
} else {
return
}
case <-ctx2.Done():
return
default:
// <-time.After(time.Duration(100) * time.Millisecond)
}
}
}()
log.Printf("wait for file fuzz(dicts:%d) %s \r", len(filedic), u)
for _, payload := range filedic {
// 接收到停止信号
if atomic.LoadInt32(&errorTimes) >= MaxErrorTimes {
Expand All @@ -235,14 +238,8 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
wg.Done() // 控制所有线程结束
<-ch // 并发控制
}()
//log.Printf("start file fuzz %s%s \r", u, payload)
for {
select {
case _, ok := <-ch:
if !ok {
stop()
return
}
case <-ctx.Done(): // 00-捕获所有线程关闭信号，并退出，close for all
atomic.AddInt32(&errorTimes, MaxErrorTimes)
return
Expand All @@ -262,9 +259,14 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
szUrl = u + payload[1:]
}
//log.Printf("start fuzz: [%s]", szUrl)
client := util.GetClient(szUrl)
if nil != client {
client.ErrCount = 0
client.ErrLimit = 999999
}
if fuzzPage, req, err := reqPage(szUrl); err == nil && nil != req && 0 < len(req.Body) {
//if 200 == req.StatusCode {
// log.Printf("%d : %s \n", req.StatusCode, szUrl)
//log.Printf("%d : %s \n", req.StatusCode, szUrl)
//}
go util.CheckHeader(req.Header, u)
// 02-状态码和req1相同，且与req1相似度>9.5，关闭所有fuzz
Expand Down Expand Up @@ -324,7 +326,10 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
}
} else { // 这里应该元子操作
if nil != err {
log.Printf("%s is err %v\n", szUrl, err)
//if nil != client && strings.Contains(err.Error(), " connect: connection reset by peer") {
// client.Client = client.GetClient(nil)
//}
//log.Printf("file fuzz %s is err %v\n", szUrl, err)
}
atomic.AddInt32(&errorTimes, 1)
}
Expand Down
13 changes: 7 additions & 6 deletions config/config.json
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"OnClient": true,
"ScanPoolSize":5000,
"JndiAddress": "https://rcejndi.51pwn.com",
"CeyeDomain": "scan4all.51pwn.com",
Expand Down Expand Up @@ -63,16 +64,16 @@
"HydraUser": "",
"HydraPass": "",
"UrlPrecise": true,
"ParseSSl": false,
"EnableSubfinder": false,
"ParseSSl": true,
"EnableSubfinder": true,
"EnableHoneyportDetection": true,
"EnableKsubdomain": true,
"KsubdomainRegxp": "([0-9a-zA-Z\\-]+\\.[0-9a-zA-Z\\-]+)$",
"naabu_dns": {},
"naabu": {"TopPorts": "1000","ScanAllIPS": true,"Threads": 50,"EnableProgressBar": false},
"priorityNmap": false,
"priorityNmap": true,
"noScan": false,
"enableMultNuclei": false,
"enableMultNuclei": true,
"enableNuclei": true,
"nuclei": {
"Severities": [5,4,3],
Expand All @@ -86,7 +87,7 @@
"HeadlessTemplateThreads": 10,
"ReportingConfig": "config/nuclei_esConfig.yaml"
},
"enablDevDebug": false,
"enablDevDebug": true,
"enablEmbedYaml": true,
"enablFileFuzz": true,
"httpx": {
Expand All @@ -100,7 +101,7 @@
"MaxRedirects": 3
},
"enableEsSv": false,
"CheckWeakPassword": true,
"CheckWeakPassword": false,
"esthread": 8,
"hydrathread": 64,
"Fuzzthreads": 16,
Expand Down
2 changes: 1 addition & 1 deletion config/databases/db_drupal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5656,7 +5656,7 @@
"515658","urlfilter"
"515659","urlicon"
"515660","urlify"
"515661","urllist"
"515661","urlList"
"515662","urlproxy"
"515663","usability_suite"
"515664","uscongress"
Expand Down
64 changes: 62 additions & 2 deletions engine/dispather.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,71 @@ package engine
import (
"github.com/hktalent/goSqlite_gorm/lib"
"github.com/hktalent/goSqlite_gorm/lib/scan/Const"
"github.com/hktalent/goSqlite_gorm/pkg/models"
"github.com/hktalent/scan4all/pkg/portScan"
)

func Dispather(task *lib.Target4Chan) {
var (
CaseScanFunc = map[int]interface{}{
Const.ScanType_SSLInfo: nil, // 01- SSL信息分析，并对域名信息进行收集、进入下一步流程
Const.ScanType_SubDomain: nil, // 02- 子域名爆破，新域名回归 到: 1 <-- -> 2，做去重处理
Const.ScanType_MergeIps: nil, // 03- 默认自动合并ip，记录ip与域名的关联关系，再发送payload时考虑：相同ip不同域名，相同payload分别发送 合并相同目标 若干域名的ip，避免扫描时重复
Const.ScanType_Pswd4hydra: nil, // 04- 密码破解，隐含包含了: 端口扫描(05-masscan + 06-nmap)
Const.ScanType_Masscan: portScan.MassScanTarget, // 05- 合并后的ip 进行快速端口扫描
Const.ScanType_Nmap: nil, // 06、精准 端口指纹，排除masscan已经识别的几种指纹
Const.ScanType_IpInfo: nil, // 07- 获取ip info
Const.ScanType_GoPoc: nil, // 08- go-poc 检测, 隐含包含了: 端口扫描(05-masscan + 06-nmap)
Const.ScanType_PortsWeb: nil, // 09- web端口识别，Naabu,识别 https，识别存活的web端口，再进入下一流程
Const.ScanType_WebFingerprints: nil, // 10- web指纹，识别蜜罐，并标识
Const.ScanType_WebDetectWaf: nil, // 11- detect WAF
Const.ScanType_WebScrapy: nil, // 12- 爬虫分析，form表单识别，字段名识别，form action提取；
Const.ScanType_WebInfo: nil, // 13- server、x-powerby、x***，url、ip、其他敏感信息（姓名、电话、地址、身份证）
Const.ScanType_WebVulsScan: nil, // 14-nuclei
Const.ScanType_WebDirScan: nil, // 14-dir爆破,Gobuster
}
)

switch Const.d {
// 扫描任务分发
// 为不同类型扫描构造参数，进行事件分发
func Dispather(task *models.Target4Chan) {
for k, _ := range CaseScanFunc {
if lib.HasScanType(task.ScanType, k) {
x1 := &models.EventData{EventType: k}
switch k {
case Const.ScanType_SSLInfo: // 01- SSL信息分析，并对域名信息进行收集、进入下一步流程
G_Engine.EventData <- x1
case Const.ScanType_SubDomain: // 02- 子域名爆破，新域名回归 到: 1 <-- -> 2，做去重处理
G_Engine.EventData <- x1
case Const.ScanType_MergeIps: // 03- 默认自动合并ip，记录ip与域名的关联关系，再发送payload时考虑：相同ip不同域名，相同payload分别发送 合并相同目标 若干域名的ip，避免扫描时重复
G_Engine.EventData <- x1
case Const.ScanType_Pswd4hydra: // 04- 密码破解，隐含包含了: 端口扫描(05-masscan + 06-nmap)
x1.EventData = []interface{}{x1, []interface{}{portScan.TargetStr(task.ScanWeb)}}
G_Engine.EventData <- x1
case Const.ScanType_Masscan: // 05- 合并后的ip 进行快速端口扫描
G_Engine.EventData <- x1
case Const.ScanType_Nmap: // 06、精准 端口指纹，排除masscan已经识别的几种指纹
G_Engine.EventData <- x1
case Const.ScanType_IpInfo: // 07- 获取ip info
G_Engine.EventData <- x1
case Const.ScanType_GoPoc: // 08- go-poc 检测, 隐含包含了: 端口扫描(05-masscan + 06-nmap)
G_Engine.EventData <- x1
case Const.ScanType_PortsWeb: // 09- web端口识别，Naabu,识别 https，识别存活的web端口，再进入下一流程
G_Engine.EventData <- x1
case Const.ScanType_WebFingerprints: // 10- web指纹，识别蜜罐，并标识
G_Engine.EventData <- x1
case Const.ScanType_WebDetectWaf: // 11- detect WAF
G_Engine.EventData <- x1
case Const.ScanType_WebScrapy: // 12- 爬虫分析，form表单识别，字段名识别，form action提取；
G_Engine.EventData <- x1
case Const.ScanType_WebInfo: // 13- server、x-powerby、x***，url、ip、其他敏感信息（姓名、电话、地址、身份证）
G_Engine.EventData <- x1
case Const.ScanType_WebVulsScan: // 14-nuclei
G_Engine.EventData <- x1
case Const.ScanType_WebDirScan: // 14-dir爆破,Gobuster
G_Engine.EventData <- x1
default:

}
}
}
}
55 changes: 25 additions & 30 deletions engine/engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,21 @@ package engine
import (
"context"
"github.com/codegangsta/inject"
"github.com/hktalent/goSqlite_gorm/pkg/models"
"github.com/hktalent/scan4all/lib/util"
"github.com/hktalent/scan4all/pkg/portScan"
"github.com/hktalent/scan4all/pocs_go"
"github.com/panjf2000/ants/v2"
"log"
"sync"
)

// 事件数据
type EventData struct {
EventType string // 类型：masscan、nmap、
EventData []interface{} // func，parms
}

// 引擎对象，全局单实例
type Engine struct {
Context *context.Context // 上下文
Wg *sync.WaitGroup // Wg
Pool int // 线程池
PoolFunc *ants.PoolWithFunc // 线程调用
EventData chan *EventData // 数据队列
Context *context.Context // 上下文
Wg *sync.WaitGroup // Wg
Pool int // 线程池
PoolFunc *ants.PoolWithFunc // 线程调用
EventData chan *models.EventData // 数据队列
}

// 全局引擎
Expand All @@ -37,11 +31,11 @@ func NewEngine(c *context.Context, pool int) *Engine {
if nil != G_Engine {
return G_Engine
}
G_Engine = &Engine{Context: c, Wg: util.Wg, Pool: pool, EventData: make(chan *EventData, pool)}
G_Engine = &Engine{Context: c, Wg: util.Wg, Pool: pool, EventData: make(chan *models.EventData, pool)}

p, err := ants.NewPoolWithFunc(pool, func(i interface{}) {
defer G_Engine.Wg.Done()
G_Engine.DoEvent(i.(*EventData))
G_Engine.DoEvent(i.(*models.EventData))
})
if nil != err {
log.Println("ants.NewPoolWithFunc is error: ", err)
Expand All @@ -59,16 +53,9 @@ func (e *Engine) Close() {
}

// case 扫描使用的函数
func (e *Engine) DoCase(ed *EventData) interface{} {
if nil != ed {
switch ed.EventType {
case "masscan":
return portScan.ScanTarget
case "nmap":
return portScan.ScanTarget
default:
return nil
}
func (e *Engine) DoCase(ed *models.EventData) interface{} {
if i, ok := CaseScanFunc[ed.EventType]; ok {
return i
}
return nil
}
Expand All @@ -77,7 +64,10 @@ func (e *Engine) DoCase(ed *EventData) interface{} {
// 每个事件自己做防重处理
// 每个事件异步执行
// 每种事件类型可以独立控制并发数
func (e *Engine) DoEvent(ed *EventData) {
func (e *Engine) DoEvent(ed *models.EventData) {
var x01 = &models.EventData{}
if nil != x01 {
}
if nil != ed {
fnCall := e.DoCase(ed)
if nil != fnCall {
Expand All @@ -87,7 +77,7 @@ func (e *Engine) DoEvent(ed *EventData) {
}
v, err := in.Invoke(fnCall)
if nil != err {
log.Printf("DoEvent %s is error: %v %+v \n", ed.EventType, err, ed.EventData)
log.Printf("DoEvent %d is error: %v %+v \n", ed.EventType, err, ed.EventData)
} else if nil != v {
log.Printf("DoEvent result %s %v\n", ed.EventType, v)
}
Expand All @@ -102,19 +92,23 @@ func init() {
util.RegInitFunc(func() {
x1 := NewEngine(&util.Ctx_global, util.GetValAsInt("ScanPoolSize", 5000))
// 异步启动一个线程处理检测，避免
util.Wg.Add(1)
go func() {
defer x1.Close()
defer func() {
x1.Close()
util.Wg.Done()
}()
//nMax := 120 // 等xxx秒都没有消息进入就退出
//nCnt := 0
for {
select {
case <-util.Ctx_global.Done():
close(util.PocCheck_pipe)
return
case x1, ok := <-G_Engine.EventData: // 各种扫描的控制
if ok {
case x2 := <-G_Engine.EventData: // 各种扫描的控制
if nil != x2 {
G_Engine.Wg.Add(1)
G_Engine.PoolFunc.Invoke(x1)
G_Engine.PoolFunc.Invoke(x2)
}
case x1, ok := <-util.PocCheck_pipe:
if util.GetValAsBool("NoPOC") || nil == x1 || !ok {
Expand All @@ -131,6 +125,7 @@ func init() {
})
default:
util.DoDelayClear()
util.DoSleep()
//var f01 float32 = float32(nCnt) / float32(nMax) * float32(100)
//fmt.Printf(" Asynchronous go PoCs detection task %%%0.2f ....\r", f01)
//<-time.After(time.Duration(1) * time.Second)
Expand Down
1 change: 1 addition & 0 deletions engine/tools.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
package engine
4 changes: 4 additions & 0 deletions fixMod.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,7 @@ git checkout vendor/github.com/projectdiscovery/nuclei/v2
go build

git add vendor
rm -rf vendor/github.com/hktalent/goSqlite_gorm
ln -s $HOME/MyWork/goSqlite_gorm $PWD/vendor/github.com/hktalent/goSqlite_gorm


1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ require (
github.com/google/go-github v17.0.0+incompatible
github.com/gorilla/websocket v1.5.0
github.com/gosnmp/gosnmp v1.35.0
github.com/hktalent/PipelineHttp v0.0.0-20221003131356-bf29cf889d54
github.com/hktalent/goSqlite_gorm v1.1.0
github.com/hktalent/jarm-go v0.0.0-20220918133110-7801447b6267
github.com/huin/asn1ber v0.0.0-20120622192748-af09f62e6358
Expand Down
Loading

0 comments on commit cc61397

Please sign in to comment.