From 742f5a0a48d5b4ef88697af6669b6a2fa14b80da Mon Sep 17 00:00:00 2001 From: Dominik Pataky Date: Mon, 6 Apr 2020 17:02:52 +0200 Subject: [PATCH] IPFIX: enhance (data|field) types and parsing; extend tests Parts of the IPFIXFieldTypes class were extracted into the new IPFIXDataTypes class, to increase readability and stability. The IPFIXDataRecord class and its field parser is now more in tune with the specifications, handling signed and unsigned, as well as float, boolean and UTF8 strings etc. Corresponding tests were extended with softflowd packets (level "ethernet") and value checks (e.g. MAC address). Resolves #25 --- netflow/ipfix.py | 186 ++++++++++++++++++++++++++++-------------- netflow/utils.py | 3 +- tests/lib.py | 75 ++++++++++++++--- tests/test_ipfix.py | 67 +++++++++++---- tests/test_netflow.py | 9 +- 5 files changed, 244 insertions(+), 96 deletions(-) diff --git a/netflow/ipfix.py b/netflow/ipfix.py index 9669aa6..f02d04c 100644 --- a/netflow/ipfix.py +++ b/netflow/ipfix.py @@ -10,7 +10,7 @@ Licensed under MIT License. See LICENSE. from collections import namedtuple import functools import struct -from typing import Optional, Union, List +from typing import Optional, Union, List, Dict FieldType = namedtuple("FieldType", ["id", "name", "type"]) DataType = namedtuple("DataType", ["type", "unpack_format"]) @@ -487,6 +487,43 @@ class IPFIXFieldTypes: (491, "bgpDestinationLargeCommunityList", "basicList"), ] + @classmethod + @functools.lru_cache + def by_id(cls, id_: int) -> Optional[FieldType]: + for item in cls.iana_field_types: + if item[0] == id_: + return FieldType(*item) + return None + + @classmethod + @functools.lru_cache + def by_name(cls, key: str) -> Optional[FieldType]: + for item in cls.iana_field_types: + if item[1] == key: + return FieldType(*item) + return None + + @classmethod + @functools.lru_cache + def get_type_unpack(cls, key: Union[int, str]) -> Optional[DataType]: + """ + This method covers the mapping from a field type to a struct.unpack format string. + BLOCKED: due to Reduced-Size Encoding, fields may be exported with a smaller length than defined in + the standard. Because of this mismatch, the parser in `IPFIXDataRecord.__init__` cannot use this method. + :param key: + :return: + """ + item = None + if type(key) == int: + item = cls.by_id(key) + elif type(key) == str: + item = cls.by_name(key) + if not item: + return None + return IPFIXDataTypes.by_name(item.type) + + +class IPFIXDataTypes: # Source: https://www.iana.org/assignments/ipfix/ipfix-information-element-data-types.csv # Reference: https://tools.ietf.org/html/rfc7011 iana_data_types = [ @@ -517,60 +554,66 @@ class IPFIXFieldTypes: # ("subTemplateMultiList", "x"), ] - @classmethod - def data_type_is_signed(cls, type_: Union[DataType, str]) -> bool: - t = type_ - if type(type_) == DataType: - t = type_.type - return t in ["signed8", "signed16", "signed32", "signed64"] - - @classmethod - def data_type_is_bytes(cls, type_: Union[DataType, str]) -> bool: - t = type_ - if type(type_) == DataType: - t = type_.type - return t in ["octetArray", "macAddress", "string", "ipv4Address", "ipv6Address"] - @classmethod @functools.lru_cache - def by_id(cls, id_: int) -> Optional[FieldType]: - for item in cls.iana_field_types: - if item[0] == id_: - return FieldType(*item) - return None - - @classmethod - @functools.lru_cache - def by_name(cls, key: str) -> Optional[FieldType]: - for item in cls.iana_field_types: - if item[1] == key: - return FieldType(*item) - return None - - @classmethod - @functools.lru_cache - def get_type_unpack(cls, key: Union[int, str]) -> Optional[DataType]: + def by_name(cls, key: str) -> Optional[DataType]: """ - This method should cover the mapping from a field type to a struct.unpack format string. - BUT: tests with softflowd v1.0.0 showed, that the template field lengths exported by softflowd differ - in some cases. Example: a field has type unsigned32, meaning 4 bytes, but the template states it is - only 2 bytes long. - Because of this mismatch, the parser in `IPFIXDataRecord.__init__` cannot use this method. + Get DataType by name if found, else None. :param key: :return: """ - item = None - if type(key) == int: - item = cls.by_id(key) - elif type(key) == str: - item = cls.by_name(key) - if not item: - return None for t in cls.iana_data_types: - if t[0] == item.type: + if t[0] == key: return DataType(*t) return None + @classmethod + def is_signed(cls, dt: Union[DataType, str]) -> bool: + """ + Check if a data type is meant to be a signed integer. + :param dt: + :return: + """ + fields = ["signed8", "signed16", "signed32", "signed64"] + if type(dt) == DataType: + return dt.type in fields + return dt in fields + + @classmethod + def is_float(cls, dt: Union[DataType, str]) -> bool: + """ + Check if data type is meant to be a float. + :param dt: + :return: + """ + fields = ["float32", "float64"] + if type(dt) == DataType: + return dt.type in fields + return dt in fields + + @classmethod + def is_bytes(cls, dt: Union[DataType, str]) -> bool: + """ + Check if a data type is meant to be parsed as bytes. + :param dt: + :return: + """ + fields = ["octetArray", "string", + "macAddress", "ipv4Address", "ipv6Address", + "dateTimeMicroseconds", "dateTimeNanoseconds"] + if type(dt) == DataType: + return dt.type in fields + return dt in fields + + @classmethod + def to_fitting_object(cls, field): + """ + Could implement conversion to IPv4Address etc. + :param field: + :return: + """ + pass + class IPFIXMalformedRecord(Exception): pass @@ -686,8 +729,7 @@ class IPFIXDataRecord: field_length = field.length offset += field_length - # Here, IPFIXFieldTypes.get_type_unpack could be used, but it seems there is a mismatch - # of field lengths between softflowd templates and IANA standards. + # Here, reduced-size encoding of fields blocks the usage of IPFIXFieldTypes.get_type_unpack. # See comment in IPFIXFieldTypes.get_type_unpack for more information. field_type: FieldType = IPFIXFieldTypes.by_id(field_type_id) @@ -696,25 +738,27 @@ class IPFIXDataRecord: # which is not standardized by IANA. raise NotImplementedError("Field type with ID {} is not implemented".format(field_type_id)) - data_type: str = field_type.type + datatype: str = field_type.type discovered_fields.append((field_type.name, field_type_id)) - if IPFIXFieldTypes.data_type_is_bytes(data_type): + + # Catch fields which are meant to be raw bytes and skip the rest + if IPFIXDataTypes.is_bytes(datatype): unpacker += "{}s".format(field_length) continue - signed = IPFIXFieldTypes.data_type_is_signed(data_type) + # Go into int, uint, float types + issigned = IPFIXDataTypes.is_signed(datatype) + isfloat = IPFIXDataTypes.is_float(datatype) + assert not(all([issigned, isfloat])) # signed int and float are exclusive + if field_length == 1: - unpacker += "b" if signed else "B" + unpacker += "b" if issigned else "B" elif field_length == 2: - unpacker += "h" if signed else "H" + unpacker += "h" if issigned else "H" elif field_length == 4: - unpacker += "i" if signed else "I" - elif field_length == 6: # MAC address, int->bytes with (value).to_bytes(6, "big") - unpacker += "6s" + unpacker += "i" if issigned else "f" if isfloat else "I" elif field_length == 8: - unpacker += "q" if signed else "Q" - elif field_length == 16: - unpacker += "16s" + unpacker += "q" if issigned else "d" if isfloat else "Q" else: raise IPFIXTemplateError("Template field_length {} not handled in unpacker".format(field_length)) @@ -724,11 +768,20 @@ class IPFIXDataRecord: # Iterate through template again, but taking the unpacked values this time for index, ((field_type_name, field_type_id), value) in enumerate(zip(discovered_fields, pack)): if type(value) is bytes: + # Check if value is raw bytes, so no conversion happened in struct.unpack if field_type_name in ["string"]: value = str(value) # TODO: handle octetArray (= does not have to be unicode encoded) + elif field_type_name in ["boolean"]: + value = True if value == 1 else False # 2 = false per RFC + elif field_type_name in ["dateTimeMicroseconds", "dateTimeNanoseconds"]: + seconds = value[:4] + fraction = value[4:] + value = (int.from_bytes(seconds, "big"), int.from_bytes(fraction, "big")) else: value = int.from_bytes(value, "big") + # If not bytes, struct.unpack already did necessary conversions (int, float...), + # value can be used as-is. self.fields.add((field_type_id, value)) self._length = offset @@ -751,7 +804,7 @@ class IPFIXSet: """A set containing the set header and a collection of records (one of templates, options, data) """ - def __init__(self, data, templates): + def __init__(self, data: bytes, templates): self.header = IPFIXSetHeader(data[0:IPFIXSetHeader.size]) self.records = [] self._templates = {} @@ -831,7 +884,7 @@ class IPFIXExportPacket: """IPFIX export packet with header, templates, options and data flowsets """ - def __init__(self, data, templates): + def __init__(self, data: bytes, templates: Dict[int, list]): self.header = IPFIXHeader(data[:IPFIXHeader.size]) self.sets = [] self._contains_new_templates = False @@ -875,9 +928,16 @@ class IPFIXExportPacket: ) -def parse_fields(data, count: int) -> (list, int): - offset = 0 - fields = [] +def parse_fields(data: bytes, count: int) -> (list, int): + """ + Parse fields from a bytes stream, based on the count of fields. + If the field is an enterprise field or not will be determinded in this function. + :param data: + :param count: + :return: List of fields and the new offset. + """ + offset: int = 0 + fields: List[Union[TemplateField, TemplateFieldEnterprise]] = [] for ctr in range(count): if data[offset] & 1 << 7 != 0: # enterprise flag set pack = struct.unpack("!HHI", data[offset:offset + 8]) diff --git a/netflow/utils.py b/netflow/utils.py index fd97831..e1ce6f9 100644 --- a/netflow/utils.py +++ b/netflow/utils.py @@ -8,6 +8,7 @@ Licensed under MIT License. See LICENSE. """ import struct +from typing import Union from .v1 import V1ExportPacket from .v5 import V5ExportPacket @@ -30,7 +31,7 @@ def get_export_version(data): return struct.unpack('!H', data[:2])[0] -def parse_packet(data, templates=None): +def parse_packet(data: Union[str, bytes], templates=None): if templates is None: # compatibility for v1 and v5 templates = {} diff --git a/tests/lib.py b/tests/lib.py index 96eee97..0d8e05e 100644 --- a/tests/lib.py +++ b/tests/lib.py @@ -87,6 +87,7 @@ def generate_packets(amount, version, template_every_x=100): # do not use random.choice - it costs performance and results in the same packet every time. def single_packet(pkts): return pkts[0] + packet_func = single_packet if len(packets) > 1: packet_func = random.choice @@ -188,23 +189,23 @@ PACKETS_V9 = [ ] # Example export for IPFIX (v10) with 4 templates, 1 option template and 8 data flow sets -PACKET_IPFIX_TEMPLATE = "000a05205e8465fd0000001300000000000200400400000e00080004000c00040016000400150004" \ +PACKET_IPFIX_TEMPLATE = "000a05202d45a4700000001300000000000200400400000e00080004000c00040016000400150004" \ "0001000400020004000a0004000e000400070002000b00020004000100060001003c000100050001" \ "000200340401000b00080004000c000400160004001500040001000400020004000a0004000e0004" \ "00200002003c000100050001000200400800000e001b0010001c0010001600040015000400010004" \ "00020004000a0004000e000400070002000b00020004000100060001003c00010005000100020034" \ "0801000b001b0010001c001000160004001500040001000400020004000a0004000e0004008b0002" \ "003c0001000500010003001e010000050001008f000400a000080131000401320004013000020100" \ - "001a00000a5900000171352e67210000000100000000000104000054976500dfac110002ff7ed688" \ - "ff7ed73a000015c70000000d000000000000000001bbe1a6061b0400ac110002976500dfff7ed688" \ + "001a00000a5900000171352e672100000001000000000001040000547f000001ac110002ff7ed688" \ + "ff7ed73a000015c70000000d000000000000000001bbe1a6061b0400ac1100027f000001ff7ed688" \ "ff7ed73a0000074f000000130000000000000000e1a601bb061f04000401004cac110002ac110001" \ "ff7db9e0ff7dc1d0000000fc00000003000000000000000008000400ac110001ac110002ff7db9e0" \ "ff7dc1d0000000fc0000000300000000000000000000040008010220fde66f14e0f1960900000242" \ "ac110002ff0200000000000000000001ff110001ff7dfad6ff7e0e95000001b00000000600000000" \ - "0000000087000600fde66f14e0f1960900000242ac110002fde66f14e0f196090000000000000001" \ + "0000000087000600fde66f14e0f196090000affeaffeaffefdabcdef123456789000000000000001" \ "ff7e567fff7e664a0000020800000005000000000000000080000600fde66f14e0f1960900000000" \ - "00000001fde66f14e0f1960900000242ac110002ff7e567fff7e664a000002080000000500000000" \ - "0000000081000600fe800000000000000042aafffe73bbfafde66f14e0f1960900000242ac110002" \ + "00000001fde66f14e0f196090000affeaffeaffeff7e567fff7e664a000002080000000500000000" \ + "0000000081000600fe800000000000000042aafffe73bbfafde66f14e0f196090000affeaffeaffe" \ "ff7e6aaaff7e6aaa0000004800000001000000000000000087000600fde66f14e0f1960900000242" \ "ac110002fe800000000000000042aafffe73bbfaff7e6aaaff7e6aaa000000400000000100000000" \ "0000000088000600fe800000000000000042acfffe110002fe800000000000000042aafffe73bbfa" \ @@ -213,19 +214,71 @@ PACKET_IPFIX_TEMPLATE = "000a05205e8465fd0000001300000000000200400400000e0008000 "0000000088000600fe800000000000000042aafffe73bbfafe800000000000000042acfffe110002" \ "ff7e92aaff7e92aa0000004800000001000000000000000087000600fe800000000000000042acff" \ "fe110002fe800000000000000042aafffe73bbfaff7e92aaff7e92aa000000400000000100000000" \ - "000000008800060008000044fde66f14e0f1960900000242ac110002fd41b7143f86000000000000" \ + "000000008800060008000044fde66f14e0f196090000affeaffeaffefd41b7143f86000000000000" \ "00000001ff7ec2a0ff7ec2a00000004a000000010000000000000000d20100351100060004000054" \ "ac1100027f000001ff7ed62eff7ed68700000036000000010000000000000000c496003511000400" \ "7f000001ac110002ff7ed62eff7ed687000000760000000100000000000000000035c49611000400" \ - "08000044fde66f14e0f1960900000242ac110002fd41b7143f8600000000000000000001ff7ef359" \ + "08000044fde66f14e0f196090000affeaffeaffefd41b7143f8600000000000000000001ff7ef359" \ "ff7ef3590000004a000000010000000000000000b1e700351100060004000054ac1100027f000001" \ "ff7f06e4ff7f06e800000036000000010000000000000000a8f90035110004007f000001ac110002" \ "ff7f06e4ff7f06e8000000a60000000100000000000000000035a8f911000400" # Example export for IPFIX with two data sets -PACKET_IPFIX = "000a00d05e8465fd00000016000000000801007cfe800000000000000042acfffe110002fde66f14" \ +PACKET_IPFIX = "000a00d02d45a47000000016000000000801007cfe800000000000000042acfffe110002fde66f14" \ "e0f196090000000000000001ff7f0755ff7f07550000004800000001000000000000000087000600" \ - "fde66f14e0f196090000000000000001fe800000000000000042acfffe110002ff7f0755ff7f0755" \ - "000000400000000100000000000000008800060008000044fde66f14e0f1960900000242ac110002" \ + "fdabcdef123456789000000000000001fe800000000000000042acfffe110002ff7f0755ff7f0755" \ + "000000400000000100000000000000008800060008000044fde66f14e0f196090000affeaffeaffe" \ "2a044e42020000000000000000000223ff7f06e9ff7f22d500000140000000040000000000000000" \ "e54c01bb06020600" + +PACKET_IPFIX_TEMPLATE_ETHER = "000a05002d45a4700000000d00000000" \ + "000200500400001200080004000c000400160004001500040001000400020004000a0004000e0004" \ + "00070002000b00020004000100060001003c000100050001003a0002003b00020038000600390006" \ + "000200440401000f00080004000c000400160004001500040001000400020004000a0004000e0004" \ + "00200002003c000100050001003a0002003b000200380006003900060002005008000012001b0010" \ + "001c001000160004001500040001000400020004000a0004000e000400070002000b000200040001" \ + "00060001003c000100050001003a0002003b00020038000600390006000200440801000f001b0010" \ + "001c001000160004001500040001000400020004000a0004000e0004008b0002003c000100050001" \ + "003a0002003b000200380006003900060003001e010000050001008f000400a00008013100040132" \ + "0004013000020100001a00000009000000b0d80a558000000001000000000001040000747f000001" \ + "ac110002e58b988be58b993e000015c70000000d000000000000000001bbe1a6061b040000000000" \ + "123456affefeaffeaffeaffeac1100027f000001e58b988be58b993e0000074f0000001300000000" \ + "00000000e1a601bb061f040000000000affeaffeaffe123456affefe0401006cac110002ac110001" \ + "e58a7be3e58a83d3000000fc0000000300000000000000000800040000000000affeaffeaffe0242" \ + "aa73bbfaac110001ac110002e58a7be3e58a83d3000000fc00000003000000000000000000000400" \ + "00000000123456affefeaffeaffeaffe080102b0fde66f14e0f196090000affeaffeaffeff020000" \ + "0000000000000001ff110001e58abcd9e58ad098000001b000000006000000000000000087000600" \ + "00000000affeaffeaffe3333ff110001fde66f14e0f196090000affeaffeaffefde66f14e0f19609" \ + "0000000000000001e58b1883e58b284e000002080000000500000000000000008000060000000000" \ + "affeaffeaffe123456affefefdabcdef123456789000000000000001fde66f14e0f1960900000242" \ + "ac110002e58b1883e58b284e0000020800000005000000000000000081000600000000000242aa73" \ + "bbfaaffeaffeaffefe800000000000000042aafffe73bbfafde66f14e0f196090000affeaffeaffe" \ + "e58b2caee58b2cae000000480000000100000000000000008700060000000000123456affefe0242" \ + "ac110002fde66f14e0f196090000affeaffeaffefe800000000000000042aafffe73bbfae58b2cae" \ + "e58b2cae000000400000000100000000000000008800060000000000affeaffeaffe123456affefe" \ + "fe800000000000000042acfffe110002fe800000000000000042aafffe73bbfae58b40aee58b40ae" \ + "000000480000000100000000000000008700060000000000affeaffeaffe123456affefefe800000" \ + "000000000042aafffe73bbfafe800000000000000042acfffe110002e58b40aee58b40ae00000040" \ + "0000000100000000000000008800060000000000123456affefeaffeaffeaffefe80000000000000" \ + "0042aafffe73bbfafe800000000000000042acfffe110002e58b54aee58b54ae0000004800000001" \ + "00000000000000008700060000000000123456affefeaffeaffeaffefe800000000000000042acff" \ + "fe110002fe800000000000000042aafffe73bbfae58b54aee58b54ae000000400000000100000000" \ + "000000008800060000000000affeaffeaffe123456affefe" + +PACKET_IPFIX_ETHER = "000a02905e8b0aa90000001600000000" \ + "08000054fde66f14e0f196090000affeaffeaffefd40abcdabcd00000000000000011111e58b84a4" \ + "e58b84a40000004a000000010000000000000000d20100351100060000000000affeaffeaffe0242" \ + "aa73bbfa04000074ac1100027f000001e58b9831e58b988a00000036000000010000000000000000" \ + "c49600351100040000000000affeaffeaffe123456affefe7f000001ac110002e58b9831e58b988a" \ + "000000760000000100000000000000000035c4961100040000000000123456affefeaffeaffeaffe" \ + "08000054fde66f14e0f196090000affeaffeaffefd40abcdabcd00000000000000011111e58bb55c" \ + "e58bb55c0000004a000000010000000000000000b1e700351100060000000000affeaffeaffe0242" \ + "aa73bbfa04000074ac1100027f000001e58bc8e8e58bc8ec00000036000000010000000000000000" \ + "a8f900351100040000000000affeaffeaffe123456affefe7f000001ac110002e58bc8e8e58bc8ec" \ + "000000a60000000100000000000000000035a8f91100040000000000123456affefeaffeaffeaffe" \ + "0801009cfe800000000000000042acfffe110002fdabcdef123456789000000000000001e58bc958" \ + "e58bc958000000480000000100000000000000008700060000000000affeaffeaffe123456affefe" \ + "fdabcdef123456789000000000000001fe800000000000000042acfffe110002e58bc958e58bc958" \ + "000000400000000100000000000000008800060000000000123456affefeaffeaffeaffe08000054" \ + "fde66f14e0f196090000affeaffeaffe2a044e42020000000000000000000223e58bc8ede58be4d8" \ + "00000140000000040000000000000000e54c01bb0602060000000000affeaffeaffe123456affefe" diff --git a/tests/test_ipfix.py b/tests/test_ipfix.py index 3f1506b..26eb863 100644 --- a/tests/test_ipfix.py +++ b/tests/test_ipfix.py @@ -11,13 +11,18 @@ Licensed under MIT License. See LICENSE. import ipaddress import unittest -from tests.lib import send_recv_packets, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX +from tests.lib import send_recv_packets, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX, PACKET_IPFIX_ETHER, \ + PACKET_IPFIX_TEMPLATE_ETHER class TestFlowExportIPFIX(unittest.TestCase): - """Test IPFIX packet parsing - """ def test_recv_ipfix_packet(self): + """ + Test general sending of raw and receiving and parsing of these packets. + If this test runs successfully, the sender thread has sent a raw bytes packet towards a locally + listening collector thread, and the collector has successfully received and parsed the packets. + :return: + """ # send packet without any template, must fail to parse (packets are queued) pkts, _, _ = send_recv_packets([PACKET_IPFIX]) self.assertEqual(len(pkts), 0) # no export is parsed due to missing template @@ -25,12 +30,31 @@ class TestFlowExportIPFIX(unittest.TestCase): # send packet with 5 templates and 20 flows, should parse correctly since the templates are known pkts, _, _ = send_recv_packets([PACKET_IPFIX_TEMPLATE]) self.assertEqual(len(pkts), 1) + p = pkts[0] self.assertEqual(p.client[0], "127.0.0.1") self.assertEqual(len(p.export.flows), 1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) # count flows self.assertEqual(len(p.export.templates), 4 + 1) # count new templates - # Inspect contents of specific flows + # send template and multiple export packets + pkts, _, _ = send_recv_packets([PACKET_IPFIX, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX]) + self.assertEqual(len(pkts), 3) + self.assertEqual(pkts[0].export.header.version, 10) + + # check amount of flows across all packets + total_flows = 0 + for packet in pkts: + total_flows += len(packet.export.flows) + self.assertEqual(total_flows, 2 + 1 + (1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) + 2 + 1) + + def test_ipfix_contents(self): + """ + Inspect content of exported flows, eg. test the value of an option flow and the correct + parsing of IPv4 and IPv6 addresses. + :return: + """ + p = send_recv_packets([PACKET_IPFIX_TEMPLATE])[0][0] + flow = p.export.flows[0] self.assertEqual(flow.meteringProcessId, 2649) self.assertEqual(flow.selectorAlgorithm, 1) @@ -43,20 +67,33 @@ class TestFlowExportIPFIX(unittest.TestCase): self.assertEqual(flow.protocolIdentifier, 6) # TCP self.assertEqual(flow.sourceTransportPort, 443) self.assertEqual(flow.destinationTransportPort, 57766) + self.assertEqual(flow.tcpControlBits, 0x1b) flow = p.export.flows[17] # IPv6 flow self.assertEqual(flow.protocolIdentifier, 17) # UDP - self.assertEqual(flow.sourceIPv6Address, 337491164212692683663430561043420610562) + self.assertEqual(flow.sourceIPv6Address, 0xfde66f14e0f196090000affeaffeaffe) self.assertEqual(ipaddress.ip_address(flow.sourceIPv6Address), # Docker ULA - ipaddress.ip_address("fde6:6f14:e0f1:9609:0:242:ac11:2")) + ipaddress.ip_address("fde6:6f14:e0f1:9609:0:affe:affe:affe")) - # send template and multiple export packets - pkts, _, _ = send_recv_packets([PACKET_IPFIX, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX]) - self.assertEqual(len(pkts), 3) - self.assertEqual(pkts[0].export.header.version, 10) + def test_ipfix_contents_ether(self): + """ + IPFIX content tests based on exports with the softflowd "-T ether" flag, meaning that layer 2 + is included in the export, like MAC addresses. + :return: + """ + pkts, _, _ = send_recv_packets([PACKET_IPFIX_TEMPLATE_ETHER, PACKET_IPFIX_ETHER]) + self.assertEqual(len(pkts), 2) + p = pkts[0] - # check amount of flows across all packets - total_flows = 0 - for packet in pkts: - total_flows += len(packet.export.flows) - self.assertEqual(total_flows, 2 + 1 + (1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) + 2 + 1) + # Inspect contents of specific flows + flow = p.export.flows[0] + self.assertEqual(flow.meteringProcessId, 9) + self.assertEqual(flow.selectorAlgorithm, 1) + self.assertEqual(flow.systemInitTimeMilliseconds, 759538800000) + + flow = p.export.flows[1] + self.assertEqual(flow.destinationIPv4Address, 2886795266) + self.assertTrue(hasattr(flow, "sourceMacAddress")) + self.assertTrue(hasattr(flow, "postDestinationMacAddress")) + self.assertEqual(flow.sourceMacAddress, 0x123456affefe) + self.assertEqual(flow.postDestinationMacAddress, 0xaffeaffeaffe) diff --git a/tests/test_netflow.py b/tests/test_netflow.py index dca9046..fcc743c 100755 --- a/tests/test_netflow.py +++ b/tests/test_netflow.py @@ -12,8 +12,9 @@ import ipaddress import random import unittest -from tests.lib import send_recv_packets, NUM_PACKETS, PACKET_INVALID, PACKET_V1, PACKET_V5, PACKET_V9_TEMPLATE, \ - PACKET_V9_TEMPLATE_MIXED, PACKETS_V9 +from tests.lib import send_recv_packets, NUM_PACKETS, \ + PACKET_INVALID, PACKET_V1, PACKET_V5, \ + PACKET_V9_TEMPLATE, PACKET_V9_TEMPLATE_MIXED, PACKETS_V9 class TestFlowExportNetflow(unittest.TestCase): @@ -150,7 +151,3 @@ class TestFlowExportNetflow(unittest.TestCase): for packet in pkts: total_flows += len(packet.export.flows) self.assertEqual(total_flows, 8 + 12 + 12 + 12) - - -if __name__ == '__main__': - unittest.main()