Improve Netflow padding detection

Author: gpotter2

scapy/layers/netflow.py

@@ -35,12 +35,31 @@
 from scapy.config import conf
 from scapy.data import IP_PROTOS
 from scapy.error import warning, Scapy_Exception
-from scapy.fields import ByteEnumField, ByteField, Field, FieldLenField, \
-    FlagsField, IPField, IntField, MACField, \
-    PacketListField, PadField, SecondsIntField, ShortEnumField, ShortField, \
-    StrField, StrFixedLenField, ThreeBytesField, UTCTimeField, XByteField, \
-    XShortField, LongField, BitField, ConditionalField, BitEnumField, \
-    StrLenField
+from scapy.fields import (
+    BitEnumField,
+    BitField,
+    ByteEnumField,
+    ByteField,
+    ConditionalField,
+    Field,
+    FieldLenField,
+    FlagsField,
+    IPField,
+    IntField,
+    LongField,
+    MACField,
+    PacketListField,
+    SecondsIntField,
+    ShortEnumField,
+    ShortField,
+    StrField,
+    StrFixedLenField,
+    StrLenField,
+    ThreeBytesField,
+    UTCTimeField,
+    XByteField,
+    XShortField,
+)
 from scapy.packet import Packet, bind_layers, bind_bottom_up
 from scapy.plist import PacketList
 from scapy.sessions import IPSession, DefaultSession
@@ -1388,14 +1407,11 @@ def default_payload_class(self, p):
 class NetflowDataflowsetV9(Packet):
     name = "Netflow DataFlowSet V9/10"
     fields_desc = [ShortField("templateID", 255),
-                   FieldLenField("length", None, length_of="records",
-                                 adjust=lambda pkt, x: x + 4 + (-x % 4)),
-                   PadField(
-                       PacketListField(
-                           "records", [],
-                           NetflowRecordV9,
-                           length_from=lambda pkt: pkt.length - 4
-                       ), 4, padwith=b"\x00")]
+                   ShortField("length", None),
+                   PacketListField(
+                       "records", [],
+                       NetflowRecordV9,
+                       length_from=lambda pkt: pkt.length - 4)]
 
     @classmethod
     def dispatch_hook(cls, _pkt=None, *args, **kargs):
@@ -1413,6 +1429,15 @@ def dispatch_hook(cls, _pkt=None, *args, **kargs):
                 return NetflowOptionsFlowset10
         return cls
 
+    def post_build(self, pkt, pay):
+        if self.length is None:
+            # Padding is optional, let's apply it on build
+            length = len(pkt)
+            pad = (-length) % 4
+            pkt = pkt[:2] + struct.pack("!H", length + pad) + pkt[4:]
+            pkt += b"\x00" * pad
+        return pkt + pay
+
 
 def _netflowv9_defragment_packet(pkt, definitions, definitions_opts, ignored):
     """Used internally to process a single packet during defragmenting"""
@@ -1467,53 +1492,56 @@ def _netflowv9_defragment_packet(pkt, definitions, definitions_opts, ignored):
             current = current.payload
     # Dissect flowsets
     if NetflowDataflowsetV9 in pkt:
-        datafl = pkt[NetflowDataflowsetV9]
-        tid = datafl.templateID
-        if tid not in definitions and tid not in definitions_opts:
-            ignored.add(tid)
-            return
-        # All data is stored in one record, awaiting to be split
-        # If fieldValue is available, the record has not been
-        # defragmented: pop it
-        try:
-            data = datafl.records[0].fieldValue
-            datafl.records.pop(0)
-        except (IndexError, AttributeError):
-            return
-        res = []
-        # Flowset record
-        # Now, according to the flow/option data,
-        # let's re-dissect NetflowDataflowsetV9
-        if tid in definitions:
-            tot_len, cls = definitions[tid]
-            while len(data) >= tot_len:
-                res.append(cls(data[:tot_len]))
-                data = data[tot_len:]
-            # Inject dissected data
-            datafl.records = res
-            if data:
-                if len(data) <= 4:
-                    datafl.add_payload(conf.padding_layer(data))
-                else:
-                    datafl.do_dissect_payload(data)
-        # Options
-        elif tid in definitions_opts:
-            (scope_len, scope_cls,
-                option_len, option_cls) = definitions_opts[tid]
-            # Dissect scopes
-            if scope_len:
-                res.append(scope_cls(data[:scope_len]))
-            if option_len:
-                res.append(
-                    option_cls(data[scope_len:scope_len + option_len])
-                )
-            if len(data) > scope_len + option_len:
-                res.append(
-                    conf.padding_layer(data[scope_len + option_len:])
-                )
-            # Inject dissected data
-            datafl.records = res
-            datafl.name = "Netflow DataFlowSet V9/10 - OPTIONS"
+        current = pkt
+        while NetflowDataflowsetV9 in current:
+            datafl = current[NetflowDataflowsetV9]
+            tid = datafl.templateID
+            if tid not in definitions and tid not in definitions_opts:
+                ignored.add(tid)
+                return
+            # All data is stored in one record, awaiting to be split
+            # If fieldValue is available, the record has not been
+            # defragmented: pop it
+            try:
+                data = datafl.records[0].fieldValue
+                datafl.records.pop(0)
+            except (IndexError, AttributeError):
+                return
+            res = []
+            # Flowset record
+            # Now, according to the flow/option data,
+            # let's re-dissect NetflowDataflowsetV9
+            if tid in definitions:
+                tot_len, cls = definitions[tid]
+                while len(data) >= tot_len:
+                    res.append(cls(data[:tot_len]))
+                    data = data[tot_len:]
+                # Inject dissected data
+                datafl.records = res
+                if data:
+                    if len(data) <= 4:
+                        datafl.add_payload(conf.padding_layer(data))
+                    else:
+                        datafl.do_dissect_payload(data)
+            # Options
+            elif tid in definitions_opts:
+                (scope_len, scope_cls,
+                    option_len, option_cls) = definitions_opts[tid]
+                # Dissect scopes
+                if scope_len:
+                    res.append(scope_cls(data[:scope_len]))
+                if option_len:
+                    res.append(
+                        option_cls(data[scope_len:scope_len + option_len])
+                    )
+                if len(data) > scope_len + option_len:
+                    res.append(
+                        conf.padding_layer(data[scope_len + option_len:])
+                    )
+                # Inject dissected data
+                datafl.records = res
+                datafl.name = "Netflow DataFlowSet V9/10 - OPTIONS"
+            current = datafl.payload
 
 
 def netflowv9_defragment(plist, verb=1):

test/scapy/layers/netflow.uts

@@ -201,6 +201,20 @@ pkt4 = NetflowTemplateV9(s)
 assert len(pkt4.template_fields) == pkt4.fieldCount
 assert sum([template.fieldLength for template in pkt4.template_fields]) == 124
 
+= NetflowV10/IPFIX - dissection without padding (GH3101)
+
+s=b'\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x08\x00E\x00\x00f\x00\x01\x00\x00@\x11|\x84\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x07\x08\x07\x00R\xee\xa2\x00\n\x00H\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x18\x01\x01\x00\x04\x00\x02\x00\x04\x00\x04\x00\x01\x00\x08\x00\x04\x00\x0c\x00\x04\x01\x01\x00\x11\x00\x00\x00\x00\x06\xc0\xa8\x00\n\xc0\xa8\x00\x0b\x01\x01\x00\x11\x00\x00\x00\x00\x06\xc0\xa8\x00\n\xc0\xa8\x00\x0b'
+pkt = netflowv9_defragment(Ether(s))[0]
+
+for i in range(1,3):
+    assert pkt.getlayer(NetflowDataflowsetV9, i).templateID == 257
+    assert pkt.getlayer(NetflowDataflowsetV9, i).records[0].IN_PKTS == b'\x00\x00\x00\x00'
+    assert pkt.getlayer(NetflowDataflowsetV9, i).records[0].PROTOCOL == 6
+    assert pkt.getlayer(NetflowDataflowsetV9, i).records[0].IPV4_SRC_ADDR == "192.168.0.10"
+    assert pkt.getlayer(NetflowDataflowsetV9, i).records[0].IPV4_DST_ADDR == "192.168.0.11"
+
+assert not pkt.getlayer(NetflowDataflowsetV9, 2).payload
+
 = NetflowV10/IPFIX - build
 
 netflow_header = NetflowHeader()/NetflowHeaderV10()