Skip to content

Commit fc8adae

Browse files
monoidicmonoidic
committed
ENH: Add new feeds to Dataplane parser, rework
1 parent 9eb388e commit fc8adae

File tree

1 file changed

+137
-56
lines changed

1 file changed

+137
-56
lines changed

‎intelmq/bots/parsers/dataplane/parser.py‎

Lines changed: 137 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -8,57 +8,135 @@
88
from intelmq.lib.bot import ParserBot
99
from intelmq.lib.message import Event
1010

11+
CATEGORY = {
12+
'dnsrd': {
13+
'classification.type': 'scanner',
14+
'protocol.application': 'dns',
15+
'event_description.text': 'Address has been seen performing a DNS recursion desired query to a remote host. '
16+
'The source report lists hosts that are suspicious of more than just port '
17+
'scanning. The host may be DNS server cataloging or searching for '
18+
'hosts to use for DNS-based DDoS amplification.',
19+
},
20+
'dnsrdany': {
21+
'classification.type': 'scanner',
22+
'protocol.application': 'dns',
23+
'event_description.text': 'Address has been seen performing a DNS recursion desired IN ANY query to a remote host. '
24+
'The source report lists hosts that are suspicious of more than just port '
25+
'scanning. The host may be DNS server cataloging or searching for '
26+
'hosts to use for DNS-based DDoS amplification.',
27+
},
28+
'dnsversion': {
29+
'classification.type': 'scanner',
30+
'protocol.application': 'dns',
31+
'event_description.text': 'Address has been seen initiating a DNS CH TXT version.bind operation to a remote host. '
32+
'The source report lists hosts that are suspicious of more than just port '
33+
'scanning. The host may be DNS server cataloging or searching for '
34+
'vulnerable DNS servers.',
35+
},
36+
'proto41': {
37+
'classification.type': 'proxy',
38+
'protocol.application': '6to4',
39+
'event_description.text': 'Address has been detected to offer open IPv6 over IPv4 tunneling. '
40+
'This could allow for the host to be used a public proxy against IPv6 '
41+
'hosts.',
42+
},
43+
'sipquery': {
44+
'classification.type': 'brute-force',
45+
'protocol.application': 'sip',
46+
'event_description.text': 'Address has been seen initiating a SIP OPTIONS query to a remote host. '
47+
'The source report lists hosts that are suspicious of more than just port '
48+
'scanning. The host may be SIP server cataloging or conducting various forms '
49+
'of telephony abuse.',
50+
},
51+
'sipinvitation': {
52+
'classification.type': 'brute-force',
53+
'protocol.application': 'sip',
54+
'event_description.text': 'Address has been seen initiating a SIP INVITE operation to a remote host. '
55+
'The source report lists hosts that are suspicious of more than just port '
56+
'scanning. The host may be SIP client cataloging or conducting various forms '
57+
'of telephony abuse.',
58+
},
59+
'sipregistration': {
60+
'classification.type': 'brute-force',
61+
'protocol.application': 'sip',
62+
'event_description.text': 'Address has been seen initiating a SIP REGISTER operation to a remote host. '
63+
'The source report lists hosts that are suspicious of more than just port '
64+
'scanning. The host may be SIP client cataloging or conducting various forms '
65+
'of telephony abuse.',
66+
},
67+
'smtpdata': {
68+
'classification.type': 'scanner',
69+
'protocol.application': 'smtp',
70+
'event_description.text': 'Address has been seen initiating a SMTP DATA operation to a remote host. '
71+
'The source report lists hosts that are suspicious of more than just port '
72+
'scanning. The host may be SMTP server cataloging or conducting various forms '
73+
'of email abuse.',
74+
},
75+
'smtpgreet': {
76+
'classification.type': 'scanner',
77+
'protocol.application': 'smtp',
78+
'event_description.text': 'Address has been seen initiating a SMTP HELO/EHLO operation to a remote host. '
79+
'The source report lists hosts that are suspicious of more than just port '
80+
'scanning. The host may be SMTP server cataloging or conducting various forms '
81+
'of email abuse.',
82+
},
83+
'sshclient': {
84+
'classification.type': 'scanner',
85+
'protocol.application': 'ssh',
86+
'event_description.text': 'Address has been seen initiating an SSH connection to a remote host. The source '
87+
'report lists hosts that are suspicious of more than just port scanning. '
88+
'The host may be SSH server cataloging or conducting authentication attack '
89+
'attempts.',
90+
},
91+
'sshpwauth': {
92+
'classification.type': 'brute-force',
93+
'protocol.application': 'ssh',
94+
'event_description.text': 'Address has been seen attempting to remotely login to a host using SSH password '
95+
'authentication. The source report lists hosts that are highly suspicious and '
96+
'are likely conducting malicious SSH password authentication attacks.',
97+
},
98+
'telnetlogin': {
99+
'classification.type': 'brute-force',
100+
'protocol.application': 'telnet',
101+
'event_description.text': 'Address has been seen initiating a telnet connection to a remote host. The source '
102+
'report lists hosts that are suspicious of more than just port scanning. '
103+
'The host may be telnet server cataloging or conducting authentication attack '
104+
'attempts.',
105+
},
106+
'vncrfb': {
107+
'classification.type': 'scanner',
108+
'protocol.application': 'vnc',
109+
'event_description.text': 'Address has been seen initiating a VNC remote buffer session to a remote host. The source '
110+
'report lists hosts that are suspicious of more than just port scanning. '
111+
'The host may be VNC/RFB server cataloging or conducting authentication attack '
112+
'attempts.',
113+
},
114+
}
11115

12-
class DataplaneParserBot(ParserBot):
13-
"""Parse the Dataplane feeds"""
14-
CATEGORY = {
15-
'sipquery': {
16-
'classification.type': 'brute-force',
17-
'protocol.application': 'sip',
18-
'event_description.text': 'Address has been seen initiating a SIP OPTIONS query to a remote host. '
19-
'The source report lists hosts that are suspicious of more than just port '
20-
'scanning. The host may be SIP server cataloging or conducting various forms '
21-
'of telephony abuse.',
22-
},
23-
'sipinvitation': {
24-
'classification.type': 'brute-force',
25-
'protocol.application': 'sip',
26-
'event_description.text': 'Address has been seen initiating a SIP INVITE operation to a remote host. '
27-
'The source report lists hosts that are suspicious of more than just port '
28-
'scanning. The host may be SIP client cataloging or conducting various forms '
29-
'of telephony abuse.',
30-
},
31-
'sipregistration': {
32-
'classification.type': 'brute-force',
33-
'protocol.application': 'sip',
34-
'event_description.text': 'Address has been seen initiating a SIP REGISTER operation to a remote host. '
35-
'The source report lists hosts that are suspicious of more than just port '
36-
'scanning. The host may be SIP client cataloging or conducting various forms '
37-
'of telephony abuse.',
38-
},
39-
'sshclient': {
40-
'classification.type': 'scanner',
41-
'protocol.application': 'ssh',
42-
'event_description.text': 'Address has been seen initiating an SSH connection to a remote host. The source '
43-
'report lists hosts that are suspicious of more than just port scanning. '
44-
'The host may be SSH server cataloging or conducting authentication attack '
45-
'attempts.',
46-
},
47-
'sshpwauth': {
48-
'classification.type': 'brute-force',
49-
'protocol.application': 'ssh',
50-
'event_description.text': 'Address has been seen attempting to remotely login to a host using SSH password '
51-
'authentication. The source report lists hosts that are highly suspicious and '
52-
'are likely conducting malicious SSH password authentication attacks.',
53-
}
54-
}
55-
56-
FILE_FORMAT = [
116+
117+
def _convert_datetime(s: str) -> str:
118+
return s.replace(' ', 'T', 1) + '+00:00'
119+
120+
121+
FILE_FORMATS = {
122+
'_default': [
123+
('source.asn', lambda x: x if x != 'NA' else None),
124+
('source.as_name', lambda x: x.split()[0] if x != 'NA' else None),
125+
('source.ip', lambda x: x),
126+
('time.source', _convert_datetime),
127+
],
128+
'proto41': [
57129
('source.asn', lambda x: x if x != 'NA' else None),
58130
('source.as_name', lambda x: x.split()[0] if x != 'NA' else None),
59131
('source.ip', lambda x: x),
60-
('time.source', lambda x: x + '+00:00'),
61-
]
132+
('extra.first_seen', _convert_datetime),
133+
('time.source', _convert_datetime),
134+
],
135+
}
136+
137+
138+
class DataplaneParserBot(ParserBot):
139+
"""Parse the Dataplane feeds"""
62140

63141
def parse_line(self, line, report):
64142
if line.startswith('#') or len(line) == 0:
@@ -67,18 +145,21 @@ def parse_line(self, line, report):
67145
event = Event(report)
68146

69147
line_contents = line.split('|')
70-
if len(line_contents) != len(self.FILE_FORMAT) + 1:
71-
raise ValueError('Incorrect format for feed {}, found line: "{}"'.format(event.get('feed.url'), line))
148+
feed_name = line_contents[-1].strip()
149+
file_format = FILE_FORMATS.get(feed_name) or FILE_FORMATS['_default']
150+
151+
if len(line_contents) != len(file_format) + 1:
152+
raise ValueError(f'Incorrect format for feed {event.get("feed.url")}, found line: "{line}"')
153+
154+
if feed_name not in CATEGORY:
155+
raise ValueError(f'Unknown data feed {feed_name}.')
72156

73-
if line_contents[-1].strip() in self.CATEGORY:
74-
event.update(self.CATEGORY[line_contents[-1].strip()])
75-
else:
76-
raise ValueError('Unknown data feed {}.'.format(line_contents[-1].strip()))
157+
event.update(CATEGORY[feed_name])
77158

78-
for field, setter in zip(line_contents, self.FILE_FORMAT):
79-
value = setter[1](field.strip())
159+
for field, (field_name, converter) in zip(line_contents, file_format):
160+
value = converter(field.strip())
80161
if value is not None:
81-
event.add(setter[0], value)
162+
event.add(field_name, value)
82163

83164
event.add('raw', line)
84165
yield event

0 commit comments

Comments
 (0)