IPFIX: enhance (data|field) types and parsing; extend tests

Parts of the IPFIXFieldTypes class were extracted into the new
IPFIXDataTypes class, to increase readability and stability.

The IPFIXDataRecord class and its field parser is now more in tune with
the specifications, handling signed and unsigned, as well as float,
boolean and UTF8 strings etc.

Corresponding tests were extended with softflowd packets (level
"ethernet") and value checks (e.g. MAC address).

Resolves #25

netflow/ipfix.py

@@ -10,7 +10,7 @@ Licensed under MIT License. See LICENSE.
 from collections import namedtuple
 import functools
 import struct
-from typing import Optional, Union, List
+from typing import Optional, Union, List, Dict
 
 FieldType = namedtuple("FieldType", ["id", "name", "type"])
 DataType = namedtuple("DataType", ["type", "unpack_format"])
@@ -487,6 +487,43 @@ class IPFIXFieldTypes:
         (491, "bgpDestinationLargeCommunityList", "basicList"),
     ]
 
+    @classmethod
+    @functools.lru_cache
+    def by_id(cls, id_: int) -> Optional[FieldType]:
+        for item in cls.iana_field_types:
+            if item[0] == id_:
+                return FieldType(*item)
+        return None
+
+    @classmethod
+    @functools.lru_cache
+    def by_name(cls, key: str) -> Optional[FieldType]:
+        for item in cls.iana_field_types:
+            if item[1] == key:
+                return FieldType(*item)
+        return None
+
+    @classmethod
+    @functools.lru_cache
+    def get_type_unpack(cls, key: Union[int, str]) -> Optional[DataType]:
+        """
+        This method covers the mapping from a field type to a struct.unpack format string.
+        BLOCKED: due to Reduced-Size Encoding, fields may be exported with a smaller length than defined in
+        the standard. Because of this mismatch, the parser in `IPFIXDataRecord.__init__` cannot use this method.
+        :param key:
+        :return:
+        """
+        item = None
+        if type(key) == int:
+            item = cls.by_id(key)
+        elif type(key) == str:
+            item = cls.by_name(key)
+        if not item:
+            return None
+        return IPFIXDataTypes.by_name(item.type)
+
+
+class IPFIXDataTypes:
     # Source: https://www.iana.org/assignments/ipfix/ipfix-information-element-data-types.csv
     # Reference: https://tools.ietf.org/html/rfc7011
     iana_data_types = [
@@ -517,60 +554,66 @@ class IPFIXFieldTypes:
         # ("subTemplateMultiList", "x"),
     ]
 
-    @classmethod
-    def data_type_is_signed(cls, type_: Union[DataType, str]) -> bool:
-        t = type_
-        if type(type_) == DataType:
-            t = type_.type
-        return t in ["signed8", "signed16", "signed32", "signed64"]
-
-    @classmethod
-    def data_type_is_bytes(cls, type_: Union[DataType, str]) -> bool:
-        t = type_
-        if type(type_) == DataType:
-            t = type_.type
-        return t in ["octetArray", "macAddress", "string", "ipv4Address", "ipv6Address"]
-
     @classmethod
     @functools.lru_cache
-    def by_id(cls, id_: int) -> Optional[FieldType]:
+    def by_name(cls, key: str) -> Optional[DataType]:
-        for item in cls.iana_field_types:
-            if item[0] == id_:
-                return FieldType(*item)
-        return None
-
-    @classmethod
-    @functools.lru_cache
-    def by_name(cls, key: str) -> Optional[FieldType]:
-        for item in cls.iana_field_types:
-            if item[1] == key:
-                return FieldType(*item)
-        return None
-
-    @classmethod
-    @functools.lru_cache
-    def get_type_unpack(cls, key: Union[int, str]) -> Optional[DataType]:
         """
-        This method should cover the mapping from a field type to a struct.unpack format string.
+        Get DataType by name if found, else None.
-        BUT: tests with softflowd v1.0.0 showed, that the template field lengths exported by softflowd differ
-        in some cases. Example: a field has type unsigned32, meaning 4 bytes, but the template states it is
-        only 2 bytes long.
-        Because of this mismatch, the parser in `IPFIXDataRecord.__init__` cannot use this method.
         :param key:
         :return:
         """
-        item = None
-        if type(key) == int:
-            item = cls.by_id(key)
-        elif type(key) == str:
-            item = cls.by_name(key)
-        if not item:
-            return None
         for t in cls.iana_data_types:
-            if t[0] == item.type:
+            if t[0] == key:
                 return DataType(*t)
         return None
 
+    @classmethod
+    def is_signed(cls, dt: Union[DataType, str]) -> bool:
+        """
+        Check if a data type is meant to be a signed integer.
+        :param dt:
+        :return:
+        """
+        fields = ["signed8", "signed16", "signed32", "signed64"]
+        if type(dt) == DataType:
+            return dt.type in fields
+        return dt in fields
+
+    @classmethod
+    def is_float(cls, dt: Union[DataType, str]) -> bool:
+        """
+        Check if data type is meant to be a float.
+        :param dt:
+        :return:
+        """
+        fields = ["float32", "float64"]
+        if type(dt) == DataType:
+            return dt.type in fields
+        return dt in fields
+
+    @classmethod
+    def is_bytes(cls, dt: Union[DataType, str]) -> bool:
+        """
+        Check if a data type is meant to be parsed as bytes.
+        :param dt:
+        :return:
+        """
+        fields = ["octetArray", "string",
+                  "macAddress", "ipv4Address", "ipv6Address",
+                  "dateTimeMicroseconds", "dateTimeNanoseconds"]
+        if type(dt) == DataType:
+            return dt.type in fields
+        return dt in fields
+
+    @classmethod
+    def to_fitting_object(cls, field):
+        """
+        Could implement conversion to IPv4Address etc.
+        :param field:
+        :return:
+        """
+        pass
+
 
 class IPFIXMalformedRecord(Exception):
     pass
@@ -686,8 +729,7 @@ class IPFIXDataRecord:
             field_length = field.length
             offset += field_length
 
-            # Here, IPFIXFieldTypes.get_type_unpack could be used, but it seems there is a mismatch
+            # Here, reduced-size encoding of fields blocks the usage of IPFIXFieldTypes.get_type_unpack.
-            # of field lengths between softflowd templates and IANA standards.
             # See comment in IPFIXFieldTypes.get_type_unpack for more information.
 
             field_type: FieldType = IPFIXFieldTypes.by_id(field_type_id)
@@ -696,25 +738,27 @@ class IPFIXDataRecord:
                 # which is not standardized by IANA.
                 raise NotImplementedError("Field type with ID {} is not implemented".format(field_type_id))
 
-            data_type: str = field_type.type
+            datatype: str = field_type.type
             discovered_fields.append((field_type.name, field_type_id))
-            if IPFIXFieldTypes.data_type_is_bytes(data_type):
+
+            # Catch fields which are meant to be raw bytes and skip the rest
+            if IPFIXDataTypes.is_bytes(datatype):
                 unpacker += "{}s".format(field_length)
                 continue
 
-            signed = IPFIXFieldTypes.data_type_is_signed(data_type)
+            # Go into int, uint, float types
+            issigned = IPFIXDataTypes.is_signed(datatype)
+            isfloat = IPFIXDataTypes.is_float(datatype)
+            assert not(all([issigned, isfloat]))  # signed int and float are exclusive
+
             if field_length == 1:
-                unpacker += "b" if signed else "B"
+                unpacker += "b" if issigned else "B"
             elif field_length == 2:
-                unpacker += "h" if signed else "H"
+                unpacker += "h" if issigned else "H"
             elif field_length == 4:
-                unpacker += "i" if signed else "I"
+                unpacker += "i" if issigned else "f" if isfloat else "I"
-            elif field_length == 6:  # MAC address, int->bytes with (value).to_bytes(6, "big")
-                unpacker += "6s"
             elif field_length == 8:
-                unpacker += "q" if signed else "Q"
+                unpacker += "q" if issigned else "d" if isfloat else "Q"
-            elif field_length == 16:
-                unpacker += "16s"
             else:
                 raise IPFIXTemplateError("Template field_length {} not handled in unpacker".format(field_length))
 
@@ -724,11 +768,20 @@ class IPFIXDataRecord:
         # Iterate through template again, but taking the unpacked values this time
         for index, ((field_type_name, field_type_id), value) in enumerate(zip(discovered_fields, pack)):
             if type(value) is bytes:
+                # Check if value is raw bytes, so no conversion happened in struct.unpack
                 if field_type_name in ["string"]:
                     value = str(value)
                 # TODO: handle octetArray (= does not have to be unicode encoded)
+                elif field_type_name in ["boolean"]:
+                    value = True if value == 1 else False  # 2 = false per RFC
+                elif field_type_name in ["dateTimeMicroseconds", "dateTimeNanoseconds"]:
+                    seconds = value[:4]
+                    fraction = value[4:]
+                    value = (int.from_bytes(seconds, "big"), int.from_bytes(fraction, "big"))
                 else:
                     value = int.from_bytes(value, "big")
+            # If not bytes, struct.unpack already did necessary conversions (int, float...),
+            # value can be used as-is.
             self.fields.add((field_type_id, value))
 
         self._length = offset
@@ -751,7 +804,7 @@ class IPFIXSet:
     """A set containing the set header and a collection of records (one of templates, options, data)
     """
 
-    def __init__(self, data, templates):
+    def __init__(self, data: bytes, templates):
         self.header = IPFIXSetHeader(data[0:IPFIXSetHeader.size])
         self.records = []
         self._templates = {}
@@ -831,7 +884,7 @@ class IPFIXExportPacket:
     """IPFIX export packet with header, templates, options and data flowsets
     """
 
-    def __init__(self, data, templates):
+    def __init__(self, data: bytes, templates: Dict[int, list]):
         self.header = IPFIXHeader(data[:IPFIXHeader.size])
         self.sets = []
         self._contains_new_templates = False
@@ -875,9 +928,16 @@ class IPFIXExportPacket:
         )
 
 
-def parse_fields(data, count: int) -> (list, int):
+def parse_fields(data: bytes, count: int) -> (list, int):
-    offset = 0
+    """
-    fields = []
+    Parse fields from a bytes stream, based on the count of fields.
+    If the field is an enterprise field or not will be determinded in this function.
+    :param data:
+    :param count:
+    :return: List of fields and the new offset.
+    """
+    offset: int = 0
+    fields: List[Union[TemplateField, TemplateFieldEnterprise]] = []
     for ctr in range(count):
         if data[offset] & 1 << 7 != 0:  # enterprise flag set
             pack = struct.unpack("!HHI", data[offset:offset + 8])

netflow/utils.py

@@ -8,6 +8,7 @@ Licensed under MIT License. See LICENSE.
 """
 
 import struct
+from typing import Union
 
 from .v1 import V1ExportPacket
 from .v5 import V5ExportPacket
@@ -30,7 +31,7 @@ def get_export_version(data):
     return struct.unpack('!H', data[:2])[0]
 
 
-def parse_packet(data, templates=None):
+def parse_packet(data: Union[str, bytes], templates=None):
     if templates is None:  # compatibility for v1 and v5
         templates = {}
 

tests/lib.py

@@ -87,6 +87,7 @@ def generate_packets(amount, version, template_every_x=100):
     # do not use random.choice - it costs performance and results in the same packet every time.
     def single_packet(pkts):
         return pkts[0]
+
     packet_func = single_packet
     if len(packets) > 1:
         packet_func = random.choice
@@ -188,23 +189,23 @@ PACKETS_V9 = [
 ]
 
 # Example export for IPFIX (v10) with 4 templates, 1 option template and 8 data flow sets
-PACKET_IPFIX_TEMPLATE = "000a05205e8465fd0000001300000000000200400400000e00080004000c00040016000400150004" \
+PACKET_IPFIX_TEMPLATE = "000a05202d45a4700000001300000000000200400400000e00080004000c00040016000400150004" \
                         "0001000400020004000a0004000e000400070002000b00020004000100060001003c000100050001" \
                         "000200340401000b00080004000c000400160004001500040001000400020004000a0004000e0004" \
                         "00200002003c000100050001000200400800000e001b0010001c0010001600040015000400010004" \
                         "00020004000a0004000e000400070002000b00020004000100060001003c00010005000100020034" \
                         "0801000b001b0010001c001000160004001500040001000400020004000a0004000e0004008b0002" \
                         "003c0001000500010003001e010000050001008f000400a000080131000401320004013000020100" \
-                        "001a00000a5900000171352e67210000000100000000000104000054976500dfac110002ff7ed688" \
+                        "001a00000a5900000171352e672100000001000000000001040000547f000001ac110002ff7ed688" \
-                        "ff7ed73a000015c70000000d000000000000000001bbe1a6061b0400ac110002976500dfff7ed688" \
+                        "ff7ed73a000015c70000000d000000000000000001bbe1a6061b0400ac1100027f000001ff7ed688" \
                         "ff7ed73a0000074f000000130000000000000000e1a601bb061f04000401004cac110002ac110001" \
                         "ff7db9e0ff7dc1d0000000fc00000003000000000000000008000400ac110001ac110002ff7db9e0" \
                         "ff7dc1d0000000fc0000000300000000000000000000040008010220fde66f14e0f1960900000242" \
                         "ac110002ff0200000000000000000001ff110001ff7dfad6ff7e0e95000001b00000000600000000" \
-                        "0000000087000600fde66f14e0f1960900000242ac110002fde66f14e0f196090000000000000001" \
+                        "0000000087000600fde66f14e0f196090000affeaffeaffefdabcdef123456789000000000000001" \
                         "ff7e567fff7e664a0000020800000005000000000000000080000600fde66f14e0f1960900000000" \
-                        "00000001fde66f14e0f1960900000242ac110002ff7e567fff7e664a000002080000000500000000" \
+                        "00000001fde66f14e0f196090000affeaffeaffeff7e567fff7e664a000002080000000500000000" \
-                        "0000000081000600fe800000000000000042aafffe73bbfafde66f14e0f1960900000242ac110002" \
+                        "0000000081000600fe800000000000000042aafffe73bbfafde66f14e0f196090000affeaffeaffe" \
                         "ff7e6aaaff7e6aaa0000004800000001000000000000000087000600fde66f14e0f1960900000242" \
                         "ac110002fe800000000000000042aafffe73bbfaff7e6aaaff7e6aaa000000400000000100000000" \
                         "0000000088000600fe800000000000000042acfffe110002fe800000000000000042aafffe73bbfa" \
@@ -213,19 +214,71 @@ PACKET_IPFIX_TEMPLATE = "000a05205e8465fd0000001300000000000200400400000e0008000
                         "0000000088000600fe800000000000000042aafffe73bbfafe800000000000000042acfffe110002" \
                         "ff7e92aaff7e92aa0000004800000001000000000000000087000600fe800000000000000042acff" \
                         "fe110002fe800000000000000042aafffe73bbfaff7e92aaff7e92aa000000400000000100000000" \
-                        "000000008800060008000044fde66f14e0f1960900000242ac110002fd41b7143f86000000000000" \
+                        "000000008800060008000044fde66f14e0f196090000affeaffeaffefd41b7143f86000000000000" \
                         "00000001ff7ec2a0ff7ec2a00000004a000000010000000000000000d20100351100060004000054" \
                         "ac1100027f000001ff7ed62eff7ed68700000036000000010000000000000000c496003511000400" \
                         "7f000001ac110002ff7ed62eff7ed687000000760000000100000000000000000035c49611000400" \
-                        "08000044fde66f14e0f1960900000242ac110002fd41b7143f8600000000000000000001ff7ef359" \
+                        "08000044fde66f14e0f196090000affeaffeaffefd41b7143f8600000000000000000001ff7ef359" \
                         "ff7ef3590000004a000000010000000000000000b1e700351100060004000054ac1100027f000001" \
                         "ff7f06e4ff7f06e800000036000000010000000000000000a8f90035110004007f000001ac110002" \
                         "ff7f06e4ff7f06e8000000a60000000100000000000000000035a8f911000400"
 
 # Example export for IPFIX with two data sets
-PACKET_IPFIX = "000a00d05e8465fd00000016000000000801007cfe800000000000000042acfffe110002fde66f14" \
+PACKET_IPFIX = "000a00d02d45a47000000016000000000801007cfe800000000000000042acfffe110002fde66f14" \
                "e0f196090000000000000001ff7f0755ff7f07550000004800000001000000000000000087000600" \
-               "fde66f14e0f196090000000000000001fe800000000000000042acfffe110002ff7f0755ff7f0755" \
+               "fdabcdef123456789000000000000001fe800000000000000042acfffe110002ff7f0755ff7f0755" \
-               "000000400000000100000000000000008800060008000044fde66f14e0f1960900000242ac110002" \
+               "000000400000000100000000000000008800060008000044fde66f14e0f196090000affeaffeaffe" \
                "2a044e42020000000000000000000223ff7f06e9ff7f22d500000140000000040000000000000000" \
                "e54c01bb06020600"
+
+PACKET_IPFIX_TEMPLATE_ETHER = "000a05002d45a4700000000d00000000" \
+                              "000200500400001200080004000c000400160004001500040001000400020004000a0004000e0004" \
+                              "00070002000b00020004000100060001003c000100050001003a0002003b00020038000600390006" \
+                              "000200440401000f00080004000c000400160004001500040001000400020004000a0004000e0004" \
+                              "00200002003c000100050001003a0002003b000200380006003900060002005008000012001b0010" \
+                              "001c001000160004001500040001000400020004000a0004000e000400070002000b000200040001" \
+                              "00060001003c000100050001003a0002003b00020038000600390006000200440801000f001b0010" \
+                              "001c001000160004001500040001000400020004000a0004000e0004008b0002003c000100050001" \
+                              "003a0002003b000200380006003900060003001e010000050001008f000400a00008013100040132" \
+                              "0004013000020100001a00000009000000b0d80a558000000001000000000001040000747f000001" \
+                              "ac110002e58b988be58b993e000015c70000000d000000000000000001bbe1a6061b040000000000" \
+                              "123456affefeaffeaffeaffeac1100027f000001e58b988be58b993e0000074f0000001300000000" \
+                              "00000000e1a601bb061f040000000000affeaffeaffe123456affefe0401006cac110002ac110001" \
+                              "e58a7be3e58a83d3000000fc0000000300000000000000000800040000000000affeaffeaffe0242" \
+                              "aa73bbfaac110001ac110002e58a7be3e58a83d3000000fc00000003000000000000000000000400" \
+                              "00000000123456affefeaffeaffeaffe080102b0fde66f14e0f196090000affeaffeaffeff020000" \
+                              "0000000000000001ff110001e58abcd9e58ad098000001b000000006000000000000000087000600" \
+                              "00000000affeaffeaffe3333ff110001fde66f14e0f196090000affeaffeaffefde66f14e0f19609" \
+                              "0000000000000001e58b1883e58b284e000002080000000500000000000000008000060000000000" \
+                              "affeaffeaffe123456affefefdabcdef123456789000000000000001fde66f14e0f1960900000242" \
+                              "ac110002e58b1883e58b284e0000020800000005000000000000000081000600000000000242aa73" \
+                              "bbfaaffeaffeaffefe800000000000000042aafffe73bbfafde66f14e0f196090000affeaffeaffe" \
+                              "e58b2caee58b2cae000000480000000100000000000000008700060000000000123456affefe0242" \
+                              "ac110002fde66f14e0f196090000affeaffeaffefe800000000000000042aafffe73bbfae58b2cae" \
+                              "e58b2cae000000400000000100000000000000008800060000000000affeaffeaffe123456affefe" \
+                              "fe800000000000000042acfffe110002fe800000000000000042aafffe73bbfae58b40aee58b40ae" \
+                              "000000480000000100000000000000008700060000000000affeaffeaffe123456affefefe800000" \
+                              "000000000042aafffe73bbfafe800000000000000042acfffe110002e58b40aee58b40ae00000040" \
+                              "0000000100000000000000008800060000000000123456affefeaffeaffeaffefe80000000000000" \
+                              "0042aafffe73bbfafe800000000000000042acfffe110002e58b54aee58b54ae0000004800000001" \
+                              "00000000000000008700060000000000123456affefeaffeaffeaffefe800000000000000042acff" \
+                              "fe110002fe800000000000000042aafffe73bbfae58b54aee58b54ae000000400000000100000000" \
+                              "000000008800060000000000affeaffeaffe123456affefe"
+
+PACKET_IPFIX_ETHER = "000a02905e8b0aa90000001600000000" \
+                     "08000054fde66f14e0f196090000affeaffeaffefd40abcdabcd00000000000000011111e58b84a4" \
+                     "e58b84a40000004a000000010000000000000000d20100351100060000000000affeaffeaffe0242" \
+                     "aa73bbfa04000074ac1100027f000001e58b9831e58b988a00000036000000010000000000000000" \
+                     "c49600351100040000000000affeaffeaffe123456affefe7f000001ac110002e58b9831e58b988a" \
+                     "000000760000000100000000000000000035c4961100040000000000123456affefeaffeaffeaffe" \
+                     "08000054fde66f14e0f196090000affeaffeaffefd40abcdabcd00000000000000011111e58bb55c" \
+                     "e58bb55c0000004a000000010000000000000000b1e700351100060000000000affeaffeaffe0242" \
+                     "aa73bbfa04000074ac1100027f000001e58bc8e8e58bc8ec00000036000000010000000000000000" \
+                     "a8f900351100040000000000affeaffeaffe123456affefe7f000001ac110002e58bc8e8e58bc8ec" \
+                     "000000a60000000100000000000000000035a8f91100040000000000123456affefeaffeaffeaffe" \
+                     "0801009cfe800000000000000042acfffe110002fdabcdef123456789000000000000001e58bc958" \
+                     "e58bc958000000480000000100000000000000008700060000000000affeaffeaffe123456affefe" \
+                     "fdabcdef123456789000000000000001fe800000000000000042acfffe110002e58bc958e58bc958" \
+                     "000000400000000100000000000000008800060000000000123456affefeaffeaffeaffe08000054" \
+                     "fde66f14e0f196090000affeaffeaffe2a044e42020000000000000000000223e58bc8ede58be4d8" \
+                     "00000140000000040000000000000000e54c01bb0602060000000000affeaffeaffe123456affefe"

tests/test_ipfix.py

@@ -11,13 +11,18 @@ Licensed under MIT License. See LICENSE.
 import ipaddress
 import unittest
 
-from tests.lib import send_recv_packets, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX
+from tests.lib import send_recv_packets, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX, PACKET_IPFIX_ETHER, \
+    PACKET_IPFIX_TEMPLATE_ETHER
 
 
 class TestFlowExportIPFIX(unittest.TestCase):
-    """Test IPFIX packet parsing
-    """
     def test_recv_ipfix_packet(self):
+        """
+        Test general sending of raw and receiving and parsing of these packets.
+        If this test runs successfully, the sender thread has sent a raw bytes packet towards a locally
+        listening collector thread, and the collector has successfully received and parsed the packets.
+        :return:
+        """
         # send packet without any template, must fail to parse (packets are queued)
         pkts, _, _ = send_recv_packets([PACKET_IPFIX])
         self.assertEqual(len(pkts), 0)  # no export is parsed due to missing template
@@ -25,12 +30,31 @@ class TestFlowExportIPFIX(unittest.TestCase):
         # send packet with 5 templates and 20 flows, should parse correctly since the templates are known
         pkts, _, _ = send_recv_packets([PACKET_IPFIX_TEMPLATE])
         self.assertEqual(len(pkts), 1)
+
         p = pkts[0]
         self.assertEqual(p.client[0], "127.0.0.1")
         self.assertEqual(len(p.export.flows), 1 + 2 + 2 + 9 + 1 + 2 + 1 + 2)  # count flows
         self.assertEqual(len(p.export.templates), 4 + 1)  # count new templates
 
-        # Inspect contents of specific flows
+        # send template and multiple export packets
+        pkts, _, _ = send_recv_packets([PACKET_IPFIX, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX])
+        self.assertEqual(len(pkts), 3)
+        self.assertEqual(pkts[0].export.header.version, 10)
+
+        # check amount of flows across all packets
+        total_flows = 0
+        for packet in pkts:
+            total_flows += len(packet.export.flows)
+        self.assertEqual(total_flows, 2 + 1 + (1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) + 2 + 1)
+
+    def test_ipfix_contents(self):
+        """
+        Inspect content of exported flows, eg. test the value of an option flow and the correct
+        parsing of IPv4 and IPv6 addresses.
+        :return:
+        """
+        p = send_recv_packets([PACKET_IPFIX_TEMPLATE])[0][0]
+
         flow = p.export.flows[0]
         self.assertEqual(flow.meteringProcessId, 2649)
         self.assertEqual(flow.selectorAlgorithm, 1)
@@ -43,20 +67,33 @@ class TestFlowExportIPFIX(unittest.TestCase):
         self.assertEqual(flow.protocolIdentifier, 6)  # TCP
         self.assertEqual(flow.sourceTransportPort, 443)
         self.assertEqual(flow.destinationTransportPort, 57766)
+        self.assertEqual(flow.tcpControlBits, 0x1b)
 
         flow = p.export.flows[17]  # IPv6 flow
         self.assertEqual(flow.protocolIdentifier, 17)  # UDP
-        self.assertEqual(flow.sourceIPv6Address, 337491164212692683663430561043420610562)
+        self.assertEqual(flow.sourceIPv6Address, 0xfde66f14e0f196090000affeaffeaffe)
         self.assertEqual(ipaddress.ip_address(flow.sourceIPv6Address),  # Docker ULA
-                         ipaddress.ip_address("fde6:6f14:e0f1:9609:0:242:ac11:2"))
+                         ipaddress.ip_address("fde6:6f14:e0f1:9609:0:affe:affe:affe"))
 
-        # send template and multiple export packets
+    def test_ipfix_contents_ether(self):
-        pkts, _, _ = send_recv_packets([PACKET_IPFIX, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX])
+        """
-        self.assertEqual(len(pkts), 3)
+        IPFIX content tests based on exports with the softflowd "-T ether" flag, meaning that layer 2
-        self.assertEqual(pkts[0].export.header.version, 10)
+        is included in the export, like MAC addresses.
+        :return:
+        """
+        pkts, _, _ = send_recv_packets([PACKET_IPFIX_TEMPLATE_ETHER, PACKET_IPFIX_ETHER])
+        self.assertEqual(len(pkts), 2)
+        p = pkts[0]
 
-        # check amount of flows across all packets
+        # Inspect contents of specific flows
-        total_flows = 0
+        flow = p.export.flows[0]
-        for packet in pkts:
+        self.assertEqual(flow.meteringProcessId, 9)
-            total_flows += len(packet.export.flows)
+        self.assertEqual(flow.selectorAlgorithm, 1)
-        self.assertEqual(total_flows, 2 + 1 + (1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) + 2 + 1)
+        self.assertEqual(flow.systemInitTimeMilliseconds, 759538800000)
+
+        flow = p.export.flows[1]
+        self.assertEqual(flow.destinationIPv4Address, 2886795266)
+        self.assertTrue(hasattr(flow, "sourceMacAddress"))
+        self.assertTrue(hasattr(flow, "postDestinationMacAddress"))
+        self.assertEqual(flow.sourceMacAddress, 0x123456affefe)
+        self.assertEqual(flow.postDestinationMacAddress, 0xaffeaffeaffe)

tests/test_netflow.py

@@ -12,8 +12,9 @@ import ipaddress
 import random
 import unittest
 
-from tests.lib import send_recv_packets, NUM_PACKETS, PACKET_INVALID, PACKET_V1, PACKET_V5, PACKET_V9_TEMPLATE, \
+from tests.lib import send_recv_packets, NUM_PACKETS, \
-    PACKET_V9_TEMPLATE_MIXED, PACKETS_V9
+    PACKET_INVALID, PACKET_V1, PACKET_V5, \
+    PACKET_V9_TEMPLATE, PACKET_V9_TEMPLATE_MIXED, PACKETS_V9
 
 
 class TestFlowExportNetflow(unittest.TestCase):
@@ -150,7 +151,3 @@ class TestFlowExportNetflow(unittest.TestCase):
         for packet in pkts:
             total_flows += len(packet.export.flows)
         self.assertEqual(total_flows, 8 + 12 + 12 + 12)
-
-
-if __name__ == '__main__':
-    unittest.main()