certtools · Aug 23, 2021 · Aug 23, 2021 · Aug 23, 2021 · Aug 23, 2021
diff --git a/docs/user/bots.rst b/docs/user/bots.rst
@@ -1624,6 +1624,7 @@ These are the supported feed name and their corresponding file name for automati
    Honeypot-Amplification-DDoS-Events        `event4_honeypot_ddos_amp`
    Honeypot-Brute-Force-Events               `event4_honeypot_brute_force`
    Honeypot-Darknet                          `event4_honeypot_darknet`
+   Honeypot-HTTP-Scan                        `event4_honeypot_http_scan`
    HTTP-Scanners                             `hp_http_scan`
    ICS-Scanners                              `hp_ics_scan`
    IP-Spoofer-Events                         `event4_ip_spoofer`

diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py
@@ -78,6 +78,8 @@
 
 """
 import re
+import base64
+import binascii
 from typing import Optional, Dict, Tuple, Any
 
 import intelmq.lib.harmonization as harmonization
@@ -204,6 +206,23 @@ def convert_date_utc(value: str) -> Optional[str]:
     return harmonization.DateTime.sanitize(value + '+00:00')
 
 
+def force_base64(value: Optional[str]) -> Optional[str]:
+    """
+    Takes input strings that may be either base64-encoded bytestrings or plaintext string,
+    and leaves the base64-encoded values untouched while encoding the non-encoded values,
+    uniformly converting the data in the field to be base64-encoded
+    """
+    if not value:
+        return None
+
+    try:
+        base64.b64decode(value, validate=True)  # return value intentionally ignored
+    except binascii.Error:
+        return base64.b64encode(value.encode()).decode()
+    else:
+        return value
+
+
 # https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-DB2
 open_db2_discovery_service = {
     'required_fields': [
@@ -2813,6 +2832,71 @@ def scan_exchange_identifier(field):
     }
 }
 
+# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-http-scanner-events/
+honeypot_http_scan = {
+    'required_fields': [
+        ('time.source', 'timestamp', add_UTC_to_timestamp),
+        ('source.ip', 'src_ip', validate_ip),
+        ('source.port', 'src_port'),
+    ],
+    'optional_fields': [
+        ('source.asn', 'src_asn', invalidate_zero),
+        ('source.geolocation.cc', 'src_geo'),
+        ('source.geolocation.region', 'src_region'),
+        ('source.geolocation.city', 'src_city'),
+        ('source.reverse_dns', 'src_hostname'),
+        ('extra.source.naics', 'src_naics', invalidate_zero),
+        ('extra.source.sector', 'src_sector', validate_to_none),
+        ('extra.', 'device_vendor', validate_to_none),
+        ('extra.', 'device_type', validate_to_none),
+        ('extra.', 'device_model', validate_to_none),
+        ('destination.ip', 'dst_ip', validate_ip),
+        ('destination.port', 'dst_port'),
+        ('destination.asn', 'dst_asn', invalidate_zero),
+        ('destination.geolocation.cc', 'dst_geo'),
+        ('destination.geolocation.region', 'dst_region'),
+        ('destination.geolocation.city', 'dst_city'),
+        ('destination.reverse_dns', 'dst_hostname'),
+        ('extra.destination.naics', 'dst_naics', invalidate_zero),
+        ('extra.destination.sector', 'dst_sector', validate_to_none),
+        ('extra.', 'public_source', validate_to_none),
+        ('malware.name', 'infection'),
+        ('extra.', 'family', validate_to_none),
+        ('extra.', 'tag', validate_to_none),
+        ('extra.', 'application', validate_to_none),
+        ('extra.', 'version', validate_to_none),
+        ('extra.', 'event_id', validate_to_none),
+        ('extra.', 'pattern', validate_to_none),
+        ('destination.url', 'http_url', convert_http_host_and_url, True),
+        ('user_agent', 'http_agent', validate_to_none),
+        ('extra.method', 'http_request_method', validate_to_none),
+        ('extra.', 'url_scheme', validate_to_none),
+        ('extra.', 'session_tags', validate_to_none),
+        ('extra.', 'vulnerability_enum', validate_to_none),
+        ('extra.', 'vulnerability_id', validate_to_none),
+        ('extra.', 'vulnerability_class', validate_to_none),
+        ('extra.', 'vulnerability_score', validate_to_none),
+        ('extra.', 'vulnerability_severity', validate_to_none),
+        ('extra.', 'vulnerability_version', validate_to_none),
+        ('extra.', 'threat_framework', validate_to_none),
+        ('extra.', 'threat_tactic_id', validate_to_none),
+        ('extra.', 'threat_technique_id', validate_to_none),
+        ('extra.', 'target_vendor', validate_to_none),
+        ('extra.', 'target_product', validate_to_none),
+        ('extra.', 'target_class', validate_to_none),
+        ('extra.', 'file_md5', validate_to_none),
+        ('extra.', 'file_sha256', validate_to_none),
+        ('extra.', 'request_raw', force_base64),
+        ('extra.', 'body_raw', force_base64),
+    ],
+    'constant_fields': {
+        'classification.identifier': 'honeypot-http-scan',
+        'classification.taxonomy': 'other',
+        'classification.type': 'other',
+        'protocol.application': 'http',
+    }
+}
+
 mapping = (
     # feed name, file name, function
     ('Accessible-ADB', 'scan_adb', accessible_adb),
@@ -2845,6 +2929,7 @@ def scan_exchange_identifier(field):
     ('Honeypot-Amplification-DDoS-Events', 'event4_honeypot_ddos_amp', honeypot_ddos_amp),
     ('Honeypot-Brute-Force-Events', 'event4_honeypot_brute_force', honeypot_brute_force),
     ('Honeypot-Darknet', 'event4_honeypot_darknet', event4_honeypot_darknet),
+    ('Honeypot-HTTP-Scan', 'event4_honeypot_http_scan', honeypot_http_scan),
     ('ICS-Scanners', 'hp_ics_scan', ics_scanners),
     ('IPv6-Sinkhole-HTTP-Drone', 'sinkhole6_http', ipv6_sinkhole_http_drone),  # legacy (replaced by event46_sinkhole_http)
     ('IP-Spoofer-Events', 'event4_ip_spoofer', event4_ip_spoofer),

diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py
@@ -0,0 +1,109 @@
+# SPDX-FileCopyrightText: 2021 Mikk Margus Möll <mikk@cert.ee>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# -*- coding: utf-8 -*-
+
+import os
+import unittest
+
+import intelmq.lib.test as test
+import intelmq.lib.utils as utils
+from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
+
+with open(os.path.join(os.path.dirname(__file__),
+                       'testdata/event4_honeypot_http_scan.csv')) as handle:
+    EXAMPLE_FILE = handle.read()
+EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
+
+EXAMPLE_REPORT = {'feed.name': 'Honeypot-HTTP-Scan',
+                  "raw": utils.base64_encode(EXAMPLE_FILE),
+                  "__type": "Report",
+                  "time.observation": "2021-08-01T12:00:00+00:00",
+                  "extra.file_name": "2021-08-01-event4_honeypot_http_scan.csv",
+                  }
+
+EVENTS = [{'__type': 'Event',
+           'feed.name': 'Honeypot-HTTP-Scan',
+           'classification.identifier': 'honeypot-http-scan',
+           'classification.taxonomy': 'other',
+           'classification.type': 'other',
+           'destination.asn': 5678,
+           'destination.geolocation.cc': 'UK',
+           'destination.geolocation.city': 'MAIDENHEAD',
+           'destination.geolocation.region': 'WINDSOR AND MAIDENHEAD',
+           'destination.ip': '109.87.65.43',
+           'destination.port': 80,
+           'extra.http_url': '/js/ueditor/wwwroot/way-board.cgi',
+           'extra.destination.naics': 518210,
+           'extra.protocol': 'tcp',
+           'extra.public_source': 'CAPRICA-EU',
+           'extra.request_raw': 'R0VUIC9qcy91ZWRpdG9yL3d3d3Jvb3Qvd2F5LWJvYXJkLmNnaSBIVFRQLzEuMHJuQWNjZXB0OiB0ZXh0L2h0bWwsYXBwbGljYXRpb24veGh0bWwreG1sLGFwcGxpY2F0aW9uL3htbDtxPTAuOSwqLyo7cT0wLjhybkFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZXJuQWNjZXB0LUxhbmd1YWdlOiBlbi1VUyxlbjtxPTAuNXJuQ29ubmVjdGlvbjogY2xvc2VybkRudDogMXJuSG9zdDogMTA5Ljg3LjY1LjQzcm5PcmlnaW46IGh0dHA6Ly8xMDkuODcuNjUuNDNyblJlZmVyZXI6IGh0dHA6Ly8xMDkuODcuNjUuNDMvcm5Vc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA2LjE7IFdPVzY0KSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvNTMuMC4yNzg1LjEwNCBTYWZhcmkvNTM3LjM2IENvcmUvMS41My4zMDg0LjQwMCBRUUJyb3dzZXIvOS42LjExMzQ2LjQwMA==',
+           'extra.source.naics': 518210,
+           'extra.source.sector': 'Communications, Service Provider, and Hosting Service',
+           'extra.version': '3.1.3-dev',
+           'malware.name': 'http-scan',
+           'protocol.application': 'http',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
+                                                 EXAMPLE_LINES[1]])),
+           'source.asn': 1234,
+           'source.geolocation.cc': 'EE',
+           'source.geolocation.city': 'TALLINN',
+           'source.geolocation.region': 'HARJUMAA',
+           'source.ip': '191.23.45.67',
+           'source.port': 36455,
+           'source.reverse_dns': '191-23-45-67-host.example.com',
+           'time.observation': '2021-08-01T12:00:00+00:00',
+           'time.source': '2021-08-01T00:24:08+00:00'},
+          {'__type': 'Event',
+           'feed.name': 'Honeypot-HTTP-Scan',
+           'classification.identifier': 'honeypot-http-scan',
+           'classification.taxonomy': 'other',
+           'classification.type': 'other',
+           'destination.asn': 23456,
+           'destination.geolocation.cc': 'UA',
+           'destination.geolocation.city': 'KHARKIV',
+           'destination.geolocation.region': "KHARKIVS'KA OBLAST'",
+           'destination.ip': '82.41.20.10',
+           'destination.port': 8080,
+           'extra.http_url': '/',
+           'extra.method': 'GET',
+           'extra.protocol': 'tcp',
+           'extra.public_source': 'CAPRICA-EU',
+           'extra.request_raw': 'R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==',
+           'extra.url_scheme': 'http',
+           'extra.user_agent': 'Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1',
+           'malware.name': 'http-scan',
+           'protocol.application': 'http',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
+                                                 EXAMPLE_LINES[2]])),
+           'source.asn': 12345,
+           'source.geolocation.cc': 'EE',
+           'source.geolocation.city': 'TALLINN',
+           'source.geolocation.region': 'HARJUMAA',
+           'source.ip': '45.67.89.123',
+           'source.port': 58610,
+           'time.observation': '2021-08-01T12:00:00+00:00',
+           'time.source': '2021-08-01T05:21:59+00:00'},
+]
+
+
+class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase):
+    """
+    A TestCase for a ShadowserverParserBot.
+    """
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = ShadowserverParserBot
+        cls.default_input_message = EXAMPLE_REPORT
+
+    def test_event(self):
+        """ Test if correct Event has been produced. """
+        self.run_bot()
+        for i, EVENT in enumerate(EVENTS):
+            self.assertMessageEqual(i, EVENT)
+
+
+if __name__ == '__main__':  # pragma: no cover
+    unittest.main()
diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv
@@ -0,0 +1,3 @@
+"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","pattern","http_url","http_agent","http_request_method","url_scheme","session_tags","vulnerability_enum","vulnerability_id","vulnerability_class","vulnerability_score","vulnerability_severity","vulnerability_version","threat_framework","threat_tactic_id","threat_technique_id","target_vendor","target_product","target_class","file_md5","file_sha256","request_raw","body_raw"
+"2021-08-01 00:24:08","tcp","191.23.45.67",36455,1234,"EE","HARJUMAA","TALLINN","191-23-45-67-host.example.com",518210,"Communications, Service Provider, and Hosting Service",,,,"109.87.65.43",80,5678,"UK","WINDSOR AND MAIDENHEAD","MAIDENHEAD",,518210,,"CAPRICA-EU","http-scan",,,,"3.1.3-dev",,"unknown","/js/ueditor/wwwroot/way-board.cgi",,,,,,,,,,,,,,,,,,,"GET /js/ueditor/wwwroot/way-board.cgi HTTP/1.0rnAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8rnAccept-Encoding: gzip, deflaternAccept-Language: en-US,en;q=0.5rnConnection: closernDnt: 1rnHost: 109.87.65.43rnOrigin: http://109.87.65.43rnReferer: http://109.87.65.43/rnUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3084.400 QQBrowser/9.6.11346.400",
+"2021-08-01 05:21:59","tcp","45.67.89.123",58610,12345,"EE","HARJUMAA","TALLINN",,,,,,,"82.41.20.10",8080,23456,"UA","KHARKIVS'KA OBLAST'","KHARKIV",,,,"CAPRICA-EU","http-scan",,,,,,,"/","Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1","GET","http",,,,,,,,,,,,,,,,"R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==",
diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license
@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2021 Mikk Margus Möll <mikk@cert.ee>
+SPDX-License-Identifier: AGPL-3.0-or-later
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		SPDX-FileCopyrightText: 2021 Mikk Margus Möll <mikk@cert.ee>
		SPDX-License-Identifier: AGPL-3.0-or-later