Merge pull request #137 from TophantTechnology/update/2.3.1

Update/2.3.1

Author: 1c3z

app/dicts/altdnsdict.txt

@@ -83,6 +83,7 @@ region
 reset
 s3
 sandbox
+scan
 scm
 search
 secure

app/dicts/blackdomain.txt

@@ -23,4 +23,5 @@
 .book.4.qq.com
 .o.tencent.com
 .o.qcloud.com
-.mur.qq.com
+.mur.qq.com
+.waf.taobao.com

app/dicts/domain_dict_test.txt

@@ -4,4 +4,5 @@ blog
 service
 mi
 ebank
+active
 un

app/modules/__init__.py

@@ -26,6 +26,7 @@ class CollectSource:
     SITESPIDER = "site_spider"
     SEARCHENGINE = "search_engine"
     MONITOR = "monitor"
+    CRTSH = "crtsh"
 
 class TaskStatus:
     WAITING = "waiting"

app/routes/export.py

@@ -7,6 +7,7 @@
 import re
 from collections import Counter
 from openpyxl.writer.excel import save_virtual_workbook
+from openpyxl.styles import Font, Color
 from app.utils import get_logger, auth
 from app import utils
 from urllib.parse import quote
@@ -149,22 +150,34 @@ def __init__(self, task_id):
         self.wb = Workbook()
         self.is_ip_task = False
 
+    def set_style(self, ws):
+        font = Font(name="Consolas", color="111111")
+        column = "ABCDEFGHIJKLMNO"
+        for x in column:
+            for y in range(1, 256):
+                ws["{}{}".format(x,y)].font = font
+
     def build_service_xl(self):
         ws = self.wb.create_sheet(title="系统服务")
         ws.column_dimensions['A'].width = 22.0
-        ws.column_dimensions['B'].width = 20.0
-        ws.column_dimensions['C'].width = 40.0
-        column_tilte = ["IP端口", "服务", "产品", "版本"]
+        ws.column_dimensions['B'].width = 10.0
+        ws.column_dimensions['C'].width = 20.0
+        ws.column_dimensions['D'].width = 40.0
+
+        column_tilte = ["IP", "端口","服务", "产品", "版本"]
         ws.append(column_tilte)
         for item in get_ip_data(self.task_id):
             for port_info in item["port_info"]:
                 row = []
-                row.append("{}:{}".format(item["ip"], port_info["port_id"]))
+                row.append(item["ip"])
+                row.append("{}".format(port_info["port_id"]))
                 row.append(port_info["service_name"])
                 row.append(port_info.get("product", ""))
                 row.append(port_info.get("version", ""))
                 ws.append(row)
 
+        self.set_style(ws)
+
     def build_ip_xl(self):
         ws = self.wb.create_sheet(title="IP")
         ws.column_dimensions['A'].width = 22.0
@@ -227,6 +240,8 @@ def build_ip_xl(self):
                 row.append(osname)
                 ws.append(row)
 
+        self.set_style(ws)
+
     def ignore_illegal(self, content):
         ILLEGAL_CHARACTERS_RE = re.compile(r'[\000-\010]|[\013-\014]|[\016-\037]')
         content = ILLEGAL_CHARACTERS_RE.sub(r'', content)
@@ -251,6 +266,8 @@ def build_site_xl(self):
             row.append(item["favicon"].get("hash", ""))
             ws.append(row)
 
+        self.set_style(ws)
+
     def build_domain_xl(self):
         ws = self.wb.create_sheet(title="域名")
         ws.column_dimensions['A'].width = 30.0
@@ -269,6 +286,8 @@ def build_domain_xl(self):
             row.append(" \r\n".join(item["ips"]))
             ws.append(row)
 
+        self.set_style(ws)
+
     def build_statist(self):
         statist = port_service_product_statist(self.task_id)
         ws = self.wb.create_sheet(title="资产统计")
@@ -334,6 +353,7 @@ def build_statist(self):
             ws["K27"] = "产品类别总数"
             ws["K28"] = product_total
 
+        self.set_style(ws)
 
     def run(self):
         task_data = get_task_data(self.task_id)
@@ -356,7 +376,8 @@ def run(self):
 
         return save_virtual_workbook(self.wb)
 
+
 def export_arl(task_id):
     task_id = task_id.strip()
     save = SaveTask(task_id)
-    return  save.run()
+    return save.run()

app/routes/policy.py

@@ -39,7 +39,8 @@ def get(self):
     "domain_brute_type": fields.String(description="域名爆破类型(big)", example="big"),
     "alt_dns": fields.Boolean(description="DNS字典智能生成", default=True),
     "riskiq_search": fields.Boolean(description="RiskIQ 调用", default=True),
-    "arl_search": fields.Boolean(description="ARL 历史查询", default=True)
+    "arl_search": fields.Boolean(description="ARL 历史查询", default=True),
+    "crtsh_search": fields.Boolean(description="crtsh 查询", default=True)
 })
 
 '''IP 相关配置选项'''

app/routes/task.py

@@ -35,7 +35,8 @@
     'options.search_engines': fields.Boolean(description="是否开启搜索引擎调用"),
     'options.site_spider': fields.Boolean(description="是否开启站点爬虫"),
     'options.riskiq_search': fields.Boolean(description="是否开启 Riskiq 调用"),
-    'options.arl_search': fields.Boolean(description="是否开启 ARL 历史查询")
+    'options.arl_search': fields.Boolean(description="是否开启 ARL 历史查询"),
+    'options.crtsh_search': fields.Boolean(description="是否开启 crt.sh 查询")
 
 }
 
@@ -65,7 +66,8 @@
     "ssl_cert": fields.Boolean(),
     "fetch_api_path": fields.Boolean(),
     "fofa_search": fields.Boolean(),
-    "sub_takeover": fields.Boolean()
+    "sub_takeover": fields.Boolean(),
+    "crtsh_search": fields.Boolean(example=True, default=True)
 })
 
 

app/routes/vuln.py

@@ -7,13 +7,11 @@
 logger = get_logger()
 
 base_search_fields = {
-    'fld': fields.String(required=False, description="IP"),
-    'site': fields.String(description="域名"),
-    'url': fields.String(required=False, description="URL"),
-    'content_length': fields.Integer(description="body 长度"),
-    'status_code': fields.Integer(description="状态码"),
-    'title': fields.String(description="标题"),
-    'source': fields.String(description="来源"),
+    'plg_name': fields.String(required=False, description="plugin ID"),
+    'plg_type': fields.String(description="类别"),
+    'vul_name': fields.String(description="漏洞名称"),
+    'app_name': fields.String(description="应用名"),
+    'target': fields.String(description="目标"),
     "task_id": fields.String(description="任务ID")
 }
 

app/services/__init__.py

@@ -18,4 +18,5 @@
 from .pageFetch import page_fetch
 from .webAppIdentify import web_app_identify
 from .syncAsset import sync_asset
-from .npoc import run_risk_cruising, run_sniffer
+from .npoc import run_risk_cruising, run_sniffer
+from .crtshClient import crtsh_search

app/services/baseThread.py

@@ -42,7 +42,7 @@ def _run(self):
                 target = target.strip()
 
             cnt += 1
-            logger.info("[{}/{}] work on {}".format(cnt, len(self.targets), target))
+            logger.debug("[{}/{}] work on {}".format(cnt, len(self.targets), target))
 
             if not target:
                 continue

app/services/crtshClient.py

@@ -0,0 +1,39 @@
+from app import utils
+logger = utils.get_logger()
+
+
+class CrtshClient:
+    def __init__(self):
+        self.url = "https://crt.sh/"
+
+    def search(self, domain):
+        param = {
+            "output": "json",
+            "q": domain
+        }
+
+        data = utils.http_req(self.url, 'get', params=param, timeout=(30.1, 50.1)).json()
+        return data
+
+
+def crtsh_search(domain):
+    name_list = []
+    try:
+        c = CrtshClient()
+        items = c.search(domain)
+        for item in items:
+            for name in item["name_value"].split():
+                name = name.strip()
+                name = name.strip("*.")
+                name = name.lower()
+                if name.endswith("."+domain):
+                    name_list.append(name)
+
+        name_list = list(set(name_list))
+        logger.info("search crtsh {} {}".format(domain, len(name_list)))
+
+    except Exception as e:
+        logger.exception(e)
+
+    return name_list
+

app/services/fetchSite.py

@@ -1,7 +1,7 @@
 import time
 from pyquery import PyQuery as pq
 import binascii
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlparse
 from urllib3.util.url import get_host
 import mmh3
 from app import utils
@@ -44,6 +44,10 @@ def work(self, site):
         if conn.status_code == 301 or conn.status_code == 302:
             url_302 = urljoin(site, conn.headers.get("Location", ""))
             if url_302 != site and url_302.startswith(site):
+                site_path = urlparse(site).path.strip("/")
+                url_302_path = urlparse(url_302).path
+                if len(site_path) > 5 and url_302_path.endswith(site_path):
+                    return
                 self.work(url_302)
 
     def run(self):

app/services/fileLeak.py

@@ -364,11 +364,6 @@ def check_page_200(self):
                 page_404 = Page(self.http_req(url_404))
                 self.page404_set.add(page_404)
 
-                if page_404.is_302():
-                    self.location_404_url.add(page_404.location_url)
-                    if page.location_url in self.location_404_url:
-                        self.page404_set.add(page)
-
                 if page_404.is_302() and page_404.location_url.endswith(page_404.url.payload + "/"):
                     self.page404_set.add(page)
                     self.skip_302 = True

app/services/npoc.py

@@ -183,7 +183,18 @@ def run_risk_cruising(plugins, targets):
 def run_sniffer(targets):
     n = NPoC(concurrency=15, tmp_dir=Config.TMP_PATH)
     x = n.plugin_name_list
-    items = n.run_poc(n.sniffer_plugin_name_set, targets)
+    new_targets = []
+
+    ##跳过80 和 443 的识别
+    for t in targets:
+        t = t.strip()
+        if t.endswith(":80"):
+            continue
+        if t.endswith(":443"):
+            continue
+        new_targets.append(t)
+
+    items = n.run_poc(n.sniffer_plugin_name_set, new_targets)
     ret = []
     for x in items:
         target = x["verify_data"]

app/services/webAnalyze.py

@@ -19,7 +19,7 @@ def work(self, site):
                           Config.DRIVER_JS ,
                           site
                           ]
-        logger.info("WebAnalyze=> {}".format(" ".join(cmd_parameters)))
+        logger.debug("WebAnalyze=> {}".format(" ".join(cmd_parameters)))
 
         output = utils.check_output(cmd_parameters, timeout=20)
         output = output.decode('utf-8')

app/tasks/domain.py

@@ -467,6 +467,9 @@ def __init__(self, base_domain = None, task_id = None, options = None):
         #用来区分是正常任务还是监控任务
         self.task_tag = "task"
 
+        #用来存放泛解析域名映射的IP
+        self._not_found_domain_ips = None
+
         self.npoc_service_target_set = set()
 
         scan_port_map = {
@@ -483,6 +486,17 @@ def __init__(self, base_domain = None, task_id = None, options = None):
         }
         self.scan_port_option = scan_port_option
 
+    @property
+    def not_found_domain_ips(self):
+        if self._not_found_domain_ips is None:
+            fake_domain = "at"+utils.random_choices(4) + "." + self.base_domain
+            self._not_found_domain_ips = utils.get_ip(fake_domain)
+
+            if self._not_found_domain_ips:
+                logger.info("not_found_domain_ips  {} {}".format(fake_domain, self._not_found_domain_ips))
+
+        return self._not_found_domain_ips
+
     def save_domain_info_list(self, domain_info_list, source = CollectSource.DOMAIN_BRUTE):
         for domain_info_obj in domain_info_list:
             domain_info = domain_info_obj.dump_json(flag=False)
@@ -515,6 +529,13 @@ def clear_domain_info_by_record(self, domain_info_list):
                 continue
 
             record = info.record_list[0]
+
+            ip = info.ip_list[0]
+
+            # 解决泛解析域名问题，果断剔除
+            if ip in self.not_found_domain_ips:
+                continue
+
             cnt = self.record_map.get(record, 0)
             cnt += 1
             self.record_map[record] = cnt
@@ -554,6 +575,20 @@ def arl_search(self):
         logger.info("end arl fetch {} {} elapse {}".format(
             self.base_domain, len(domain_info_list), elapse))
 
+    def crtsh_search(self):
+        t1 = time.time()
+        logger.info("start crtsh search {}".format(self.base_domain))
+        crtsh_domains = services.crtsh_search(self.base_domain)
+        domain_info_list = self.build_domain_info(crtsh_domains)
+        if self.task_tag == "task":
+            domain_info_list = self.clear_domain_info_by_record(domain_info_list)
+            self.save_domain_info_list(domain_info_list, source=CollectSource.CRTSH)
+
+        self.domain_info_list.extend(domain_info_list)
+        elapse = time.time() - t1
+        logger.info("end crtsh search {} {} elapse {}".format(
+            self.base_domain, len(domain_info_list), elapse))
+
     def build_domain_info(self, domains):
         """
         构建domain_info_list 带去重功能
@@ -900,14 +935,22 @@ def domain_fetch(self):
                 self.domain_info_list.append(domain_info)
                 self.save_domain_info_list([domain_info])
 
-        '''***RiskIQ查询****'''
+        # ***RiskIQ查询****
         if self.options.get("riskiq_search") and services.riskiq_quota() > 0:
             self.update_task_field("status", "riskiq_search")
             t1 = time.time()
             self.riskiq_search()
             elapse = time.time() - t1
             self.update_services("riskiq_search", elapse)
 
+        # crt.sh 网站查询
+        if self.options.get("crtsh_search"):
+            self.update_task_field("status", "crtsh_search")
+            t1 = time.time()
+            self.crtsh_search()
+            elapse = time.time() - t1
+            self.update_services("crtsh_search", elapse)
+
         if self.options.get("arl_search"):
             self.update_task_field("status", "arl_search")
             t1 = time.time()
@@ -1084,6 +1127,7 @@ def brute_config(self):
             item["save_date"] = utils.curr_date()
             utils.conn_db('vuln').insert_one(item)
 
+
     def run(self):
 
         self.update_task_field("start_time", utils.curr_date())
@@ -1133,7 +1177,8 @@ def add_domain_to_scope(domain, scope_id):
             'site_spider': False,
             'search_engines': False,
             'ssl_cert': False,
-            'fofa_search': False
+            'fofa_search': False,
+            'crtsh_search': True
         }
     }