IPFIX: enhance (data|field) types and parsing; extend tests
Parts of the IPFIXFieldTypes class were extracted into the new IPFIXDataTypes class, to increase readability and stability. The IPFIXDataRecord class and its field parser is now more in tune with the specifications, handling signed and unsigned, as well as float, boolean and UTF8 strings etc. Corresponding tests were extended with softflowd packets (level "ethernet") and value checks (e.g. MAC address). Resolves #25
This commit is contained in:
parent
405f9c6a67
commit
742f5a0a48
186
netflow/ipfix.py
186
netflow/ipfix.py
|
@ -10,7 +10,7 @@ Licensed under MIT License. See LICENSE.
|
||||||
from collections import namedtuple
|
from collections import namedtuple
|
||||||
import functools
|
import functools
|
||||||
import struct
|
import struct
|
||||||
from typing import Optional, Union, List
|
from typing import Optional, Union, List, Dict
|
||||||
|
|
||||||
FieldType = namedtuple("FieldType", ["id", "name", "type"])
|
FieldType = namedtuple("FieldType", ["id", "name", "type"])
|
||||||
DataType = namedtuple("DataType", ["type", "unpack_format"])
|
DataType = namedtuple("DataType", ["type", "unpack_format"])
|
||||||
|
@ -487,6 +487,43 @@ class IPFIXFieldTypes:
|
||||||
(491, "bgpDestinationLargeCommunityList", "basicList"),
|
(491, "bgpDestinationLargeCommunityList", "basicList"),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@functools.lru_cache
|
||||||
|
def by_id(cls, id_: int) -> Optional[FieldType]:
|
||||||
|
for item in cls.iana_field_types:
|
||||||
|
if item[0] == id_:
|
||||||
|
return FieldType(*item)
|
||||||
|
return None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@functools.lru_cache
|
||||||
|
def by_name(cls, key: str) -> Optional[FieldType]:
|
||||||
|
for item in cls.iana_field_types:
|
||||||
|
if item[1] == key:
|
||||||
|
return FieldType(*item)
|
||||||
|
return None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@functools.lru_cache
|
||||||
|
def get_type_unpack(cls, key: Union[int, str]) -> Optional[DataType]:
|
||||||
|
"""
|
||||||
|
This method covers the mapping from a field type to a struct.unpack format string.
|
||||||
|
BLOCKED: due to Reduced-Size Encoding, fields may be exported with a smaller length than defined in
|
||||||
|
the standard. Because of this mismatch, the parser in `IPFIXDataRecord.__init__` cannot use this method.
|
||||||
|
:param key:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
item = None
|
||||||
|
if type(key) == int:
|
||||||
|
item = cls.by_id(key)
|
||||||
|
elif type(key) == str:
|
||||||
|
item = cls.by_name(key)
|
||||||
|
if not item:
|
||||||
|
return None
|
||||||
|
return IPFIXDataTypes.by_name(item.type)
|
||||||
|
|
||||||
|
|
||||||
|
class IPFIXDataTypes:
|
||||||
# Source: https://www.iana.org/assignments/ipfix/ipfix-information-element-data-types.csv
|
# Source: https://www.iana.org/assignments/ipfix/ipfix-information-element-data-types.csv
|
||||||
# Reference: https://tools.ietf.org/html/rfc7011
|
# Reference: https://tools.ietf.org/html/rfc7011
|
||||||
iana_data_types = [
|
iana_data_types = [
|
||||||
|
@ -517,60 +554,66 @@ class IPFIXFieldTypes:
|
||||||
# ("subTemplateMultiList", "x"),
|
# ("subTemplateMultiList", "x"),
|
||||||
]
|
]
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def data_type_is_signed(cls, type_: Union[DataType, str]) -> bool:
|
|
||||||
t = type_
|
|
||||||
if type(type_) == DataType:
|
|
||||||
t = type_.type
|
|
||||||
return t in ["signed8", "signed16", "signed32", "signed64"]
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def data_type_is_bytes(cls, type_: Union[DataType, str]) -> bool:
|
|
||||||
t = type_
|
|
||||||
if type(type_) == DataType:
|
|
||||||
t = type_.type
|
|
||||||
return t in ["octetArray", "macAddress", "string", "ipv4Address", "ipv6Address"]
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@functools.lru_cache
|
@functools.lru_cache
|
||||||
def by_id(cls, id_: int) -> Optional[FieldType]:
|
def by_name(cls, key: str) -> Optional[DataType]:
|
||||||
for item in cls.iana_field_types:
|
|
||||||
if item[0] == id_:
|
|
||||||
return FieldType(*item)
|
|
||||||
return None
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
@functools.lru_cache
|
|
||||||
def by_name(cls, key: str) -> Optional[FieldType]:
|
|
||||||
for item in cls.iana_field_types:
|
|
||||||
if item[1] == key:
|
|
||||||
return FieldType(*item)
|
|
||||||
return None
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
@functools.lru_cache
|
|
||||||
def get_type_unpack(cls, key: Union[int, str]) -> Optional[DataType]:
|
|
||||||
"""
|
"""
|
||||||
This method should cover the mapping from a field type to a struct.unpack format string.
|
Get DataType by name if found, else None.
|
||||||
BUT: tests with softflowd v1.0.0 showed, that the template field lengths exported by softflowd differ
|
|
||||||
in some cases. Example: a field has type unsigned32, meaning 4 bytes, but the template states it is
|
|
||||||
only 2 bytes long.
|
|
||||||
Because of this mismatch, the parser in `IPFIXDataRecord.__init__` cannot use this method.
|
|
||||||
:param key:
|
:param key:
|
||||||
:return:
|
:return:
|
||||||
"""
|
"""
|
||||||
item = None
|
|
||||||
if type(key) == int:
|
|
||||||
item = cls.by_id(key)
|
|
||||||
elif type(key) == str:
|
|
||||||
item = cls.by_name(key)
|
|
||||||
if not item:
|
|
||||||
return None
|
|
||||||
for t in cls.iana_data_types:
|
for t in cls.iana_data_types:
|
||||||
if t[0] == item.type:
|
if t[0] == key:
|
||||||
return DataType(*t)
|
return DataType(*t)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def is_signed(cls, dt: Union[DataType, str]) -> bool:
|
||||||
|
"""
|
||||||
|
Check if a data type is meant to be a signed integer.
|
||||||
|
:param dt:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
fields = ["signed8", "signed16", "signed32", "signed64"]
|
||||||
|
if type(dt) == DataType:
|
||||||
|
return dt.type in fields
|
||||||
|
return dt in fields
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def is_float(cls, dt: Union[DataType, str]) -> bool:
|
||||||
|
"""
|
||||||
|
Check if data type is meant to be a float.
|
||||||
|
:param dt:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
fields = ["float32", "float64"]
|
||||||
|
if type(dt) == DataType:
|
||||||
|
return dt.type in fields
|
||||||
|
return dt in fields
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def is_bytes(cls, dt: Union[DataType, str]) -> bool:
|
||||||
|
"""
|
||||||
|
Check if a data type is meant to be parsed as bytes.
|
||||||
|
:param dt:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
fields = ["octetArray", "string",
|
||||||
|
"macAddress", "ipv4Address", "ipv6Address",
|
||||||
|
"dateTimeMicroseconds", "dateTimeNanoseconds"]
|
||||||
|
if type(dt) == DataType:
|
||||||
|
return dt.type in fields
|
||||||
|
return dt in fields
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def to_fitting_object(cls, field):
|
||||||
|
"""
|
||||||
|
Could implement conversion to IPv4Address etc.
|
||||||
|
:param field:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class IPFIXMalformedRecord(Exception):
|
class IPFIXMalformedRecord(Exception):
|
||||||
pass
|
pass
|
||||||
|
@ -686,8 +729,7 @@ class IPFIXDataRecord:
|
||||||
field_length = field.length
|
field_length = field.length
|
||||||
offset += field_length
|
offset += field_length
|
||||||
|
|
||||||
# Here, IPFIXFieldTypes.get_type_unpack could be used, but it seems there is a mismatch
|
# Here, reduced-size encoding of fields blocks the usage of IPFIXFieldTypes.get_type_unpack.
|
||||||
# of field lengths between softflowd templates and IANA standards.
|
|
||||||
# See comment in IPFIXFieldTypes.get_type_unpack for more information.
|
# See comment in IPFIXFieldTypes.get_type_unpack for more information.
|
||||||
|
|
||||||
field_type: FieldType = IPFIXFieldTypes.by_id(field_type_id)
|
field_type: FieldType = IPFIXFieldTypes.by_id(field_type_id)
|
||||||
|
@ -696,25 +738,27 @@ class IPFIXDataRecord:
|
||||||
# which is not standardized by IANA.
|
# which is not standardized by IANA.
|
||||||
raise NotImplementedError("Field type with ID {} is not implemented".format(field_type_id))
|
raise NotImplementedError("Field type with ID {} is not implemented".format(field_type_id))
|
||||||
|
|
||||||
data_type: str = field_type.type
|
datatype: str = field_type.type
|
||||||
discovered_fields.append((field_type.name, field_type_id))
|
discovered_fields.append((field_type.name, field_type_id))
|
||||||
if IPFIXFieldTypes.data_type_is_bytes(data_type):
|
|
||||||
|
# Catch fields which are meant to be raw bytes and skip the rest
|
||||||
|
if IPFIXDataTypes.is_bytes(datatype):
|
||||||
unpacker += "{}s".format(field_length)
|
unpacker += "{}s".format(field_length)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
signed = IPFIXFieldTypes.data_type_is_signed(data_type)
|
# Go into int, uint, float types
|
||||||
|
issigned = IPFIXDataTypes.is_signed(datatype)
|
||||||
|
isfloat = IPFIXDataTypes.is_float(datatype)
|
||||||
|
assert not(all([issigned, isfloat])) # signed int and float are exclusive
|
||||||
|
|
||||||
if field_length == 1:
|
if field_length == 1:
|
||||||
unpacker += "b" if signed else "B"
|
unpacker += "b" if issigned else "B"
|
||||||
elif field_length == 2:
|
elif field_length == 2:
|
||||||
unpacker += "h" if signed else "H"
|
unpacker += "h" if issigned else "H"
|
||||||
elif field_length == 4:
|
elif field_length == 4:
|
||||||
unpacker += "i" if signed else "I"
|
unpacker += "i" if issigned else "f" if isfloat else "I"
|
||||||
elif field_length == 6: # MAC address, int->bytes with (value).to_bytes(6, "big")
|
|
||||||
unpacker += "6s"
|
|
||||||
elif field_length == 8:
|
elif field_length == 8:
|
||||||
unpacker += "q" if signed else "Q"
|
unpacker += "q" if issigned else "d" if isfloat else "Q"
|
||||||
elif field_length == 16:
|
|
||||||
unpacker += "16s"
|
|
||||||
else:
|
else:
|
||||||
raise IPFIXTemplateError("Template field_length {} not handled in unpacker".format(field_length))
|
raise IPFIXTemplateError("Template field_length {} not handled in unpacker".format(field_length))
|
||||||
|
|
||||||
|
@ -724,11 +768,20 @@ class IPFIXDataRecord:
|
||||||
# Iterate through template again, but taking the unpacked values this time
|
# Iterate through template again, but taking the unpacked values this time
|
||||||
for index, ((field_type_name, field_type_id), value) in enumerate(zip(discovered_fields, pack)):
|
for index, ((field_type_name, field_type_id), value) in enumerate(zip(discovered_fields, pack)):
|
||||||
if type(value) is bytes:
|
if type(value) is bytes:
|
||||||
|
# Check if value is raw bytes, so no conversion happened in struct.unpack
|
||||||
if field_type_name in ["string"]:
|
if field_type_name in ["string"]:
|
||||||
value = str(value)
|
value = str(value)
|
||||||
# TODO: handle octetArray (= does not have to be unicode encoded)
|
# TODO: handle octetArray (= does not have to be unicode encoded)
|
||||||
|
elif field_type_name in ["boolean"]:
|
||||||
|
value = True if value == 1 else False # 2 = false per RFC
|
||||||
|
elif field_type_name in ["dateTimeMicroseconds", "dateTimeNanoseconds"]:
|
||||||
|
seconds = value[:4]
|
||||||
|
fraction = value[4:]
|
||||||
|
value = (int.from_bytes(seconds, "big"), int.from_bytes(fraction, "big"))
|
||||||
else:
|
else:
|
||||||
value = int.from_bytes(value, "big")
|
value = int.from_bytes(value, "big")
|
||||||
|
# If not bytes, struct.unpack already did necessary conversions (int, float...),
|
||||||
|
# value can be used as-is.
|
||||||
self.fields.add((field_type_id, value))
|
self.fields.add((field_type_id, value))
|
||||||
|
|
||||||
self._length = offset
|
self._length = offset
|
||||||
|
@ -751,7 +804,7 @@ class IPFIXSet:
|
||||||
"""A set containing the set header and a collection of records (one of templates, options, data)
|
"""A set containing the set header and a collection of records (one of templates, options, data)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, data, templates):
|
def __init__(self, data: bytes, templates):
|
||||||
self.header = IPFIXSetHeader(data[0:IPFIXSetHeader.size])
|
self.header = IPFIXSetHeader(data[0:IPFIXSetHeader.size])
|
||||||
self.records = []
|
self.records = []
|
||||||
self._templates = {}
|
self._templates = {}
|
||||||
|
@ -831,7 +884,7 @@ class IPFIXExportPacket:
|
||||||
"""IPFIX export packet with header, templates, options and data flowsets
|
"""IPFIX export packet with header, templates, options and data flowsets
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, data, templates):
|
def __init__(self, data: bytes, templates: Dict[int, list]):
|
||||||
self.header = IPFIXHeader(data[:IPFIXHeader.size])
|
self.header = IPFIXHeader(data[:IPFIXHeader.size])
|
||||||
self.sets = []
|
self.sets = []
|
||||||
self._contains_new_templates = False
|
self._contains_new_templates = False
|
||||||
|
@ -875,9 +928,16 @@ class IPFIXExportPacket:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def parse_fields(data, count: int) -> (list, int):
|
def parse_fields(data: bytes, count: int) -> (list, int):
|
||||||
offset = 0
|
"""
|
||||||
fields = []
|
Parse fields from a bytes stream, based on the count of fields.
|
||||||
|
If the field is an enterprise field or not will be determinded in this function.
|
||||||
|
:param data:
|
||||||
|
:param count:
|
||||||
|
:return: List of fields and the new offset.
|
||||||
|
"""
|
||||||
|
offset: int = 0
|
||||||
|
fields: List[Union[TemplateField, TemplateFieldEnterprise]] = []
|
||||||
for ctr in range(count):
|
for ctr in range(count):
|
||||||
if data[offset] & 1 << 7 != 0: # enterprise flag set
|
if data[offset] & 1 << 7 != 0: # enterprise flag set
|
||||||
pack = struct.unpack("!HHI", data[offset:offset + 8])
|
pack = struct.unpack("!HHI", data[offset:offset + 8])
|
||||||
|
|
|
@ -8,6 +8,7 @@ Licensed under MIT License. See LICENSE.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import struct
|
import struct
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
from .v1 import V1ExportPacket
|
from .v1 import V1ExportPacket
|
||||||
from .v5 import V5ExportPacket
|
from .v5 import V5ExportPacket
|
||||||
|
@ -30,7 +31,7 @@ def get_export_version(data):
|
||||||
return struct.unpack('!H', data[:2])[0]
|
return struct.unpack('!H', data[:2])[0]
|
||||||
|
|
||||||
|
|
||||||
def parse_packet(data, templates=None):
|
def parse_packet(data: Union[str, bytes], templates=None):
|
||||||
if templates is None: # compatibility for v1 and v5
|
if templates is None: # compatibility for v1 and v5
|
||||||
templates = {}
|
templates = {}
|
||||||
|
|
||||||
|
|
75
tests/lib.py
75
tests/lib.py
|
@ -87,6 +87,7 @@ def generate_packets(amount, version, template_every_x=100):
|
||||||
# do not use random.choice - it costs performance and results in the same packet every time.
|
# do not use random.choice - it costs performance and results in the same packet every time.
|
||||||
def single_packet(pkts):
|
def single_packet(pkts):
|
||||||
return pkts[0]
|
return pkts[0]
|
||||||
|
|
||||||
packet_func = single_packet
|
packet_func = single_packet
|
||||||
if len(packets) > 1:
|
if len(packets) > 1:
|
||||||
packet_func = random.choice
|
packet_func = random.choice
|
||||||
|
@ -188,23 +189,23 @@ PACKETS_V9 = [
|
||||||
]
|
]
|
||||||
|
|
||||||
# Example export for IPFIX (v10) with 4 templates, 1 option template and 8 data flow sets
|
# Example export for IPFIX (v10) with 4 templates, 1 option template and 8 data flow sets
|
||||||
PACKET_IPFIX_TEMPLATE = "000a05205e8465fd0000001300000000000200400400000e00080004000c00040016000400150004" \
|
PACKET_IPFIX_TEMPLATE = "000a05202d45a4700000001300000000000200400400000e00080004000c00040016000400150004" \
|
||||||
"0001000400020004000a0004000e000400070002000b00020004000100060001003c000100050001" \
|
"0001000400020004000a0004000e000400070002000b00020004000100060001003c000100050001" \
|
||||||
"000200340401000b00080004000c000400160004001500040001000400020004000a0004000e0004" \
|
"000200340401000b00080004000c000400160004001500040001000400020004000a0004000e0004" \
|
||||||
"00200002003c000100050001000200400800000e001b0010001c0010001600040015000400010004" \
|
"00200002003c000100050001000200400800000e001b0010001c0010001600040015000400010004" \
|
||||||
"00020004000a0004000e000400070002000b00020004000100060001003c00010005000100020034" \
|
"00020004000a0004000e000400070002000b00020004000100060001003c00010005000100020034" \
|
||||||
"0801000b001b0010001c001000160004001500040001000400020004000a0004000e0004008b0002" \
|
"0801000b001b0010001c001000160004001500040001000400020004000a0004000e0004008b0002" \
|
||||||
"003c0001000500010003001e010000050001008f000400a000080131000401320004013000020100" \
|
"003c0001000500010003001e010000050001008f000400a000080131000401320004013000020100" \
|
||||||
"001a00000a5900000171352e67210000000100000000000104000054976500dfac110002ff7ed688" \
|
"001a00000a5900000171352e672100000001000000000001040000547f000001ac110002ff7ed688" \
|
||||||
"ff7ed73a000015c70000000d000000000000000001bbe1a6061b0400ac110002976500dfff7ed688" \
|
"ff7ed73a000015c70000000d000000000000000001bbe1a6061b0400ac1100027f000001ff7ed688" \
|
||||||
"ff7ed73a0000074f000000130000000000000000e1a601bb061f04000401004cac110002ac110001" \
|
"ff7ed73a0000074f000000130000000000000000e1a601bb061f04000401004cac110002ac110001" \
|
||||||
"ff7db9e0ff7dc1d0000000fc00000003000000000000000008000400ac110001ac110002ff7db9e0" \
|
"ff7db9e0ff7dc1d0000000fc00000003000000000000000008000400ac110001ac110002ff7db9e0" \
|
||||||
"ff7dc1d0000000fc0000000300000000000000000000040008010220fde66f14e0f1960900000242" \
|
"ff7dc1d0000000fc0000000300000000000000000000040008010220fde66f14e0f1960900000242" \
|
||||||
"ac110002ff0200000000000000000001ff110001ff7dfad6ff7e0e95000001b00000000600000000" \
|
"ac110002ff0200000000000000000001ff110001ff7dfad6ff7e0e95000001b00000000600000000" \
|
||||||
"0000000087000600fde66f14e0f1960900000242ac110002fde66f14e0f196090000000000000001" \
|
"0000000087000600fde66f14e0f196090000affeaffeaffefdabcdef123456789000000000000001" \
|
||||||
"ff7e567fff7e664a0000020800000005000000000000000080000600fde66f14e0f1960900000000" \
|
"ff7e567fff7e664a0000020800000005000000000000000080000600fde66f14e0f1960900000000" \
|
||||||
"00000001fde66f14e0f1960900000242ac110002ff7e567fff7e664a000002080000000500000000" \
|
"00000001fde66f14e0f196090000affeaffeaffeff7e567fff7e664a000002080000000500000000" \
|
||||||
"0000000081000600fe800000000000000042aafffe73bbfafde66f14e0f1960900000242ac110002" \
|
"0000000081000600fe800000000000000042aafffe73bbfafde66f14e0f196090000affeaffeaffe" \
|
||||||
"ff7e6aaaff7e6aaa0000004800000001000000000000000087000600fde66f14e0f1960900000242" \
|
"ff7e6aaaff7e6aaa0000004800000001000000000000000087000600fde66f14e0f1960900000242" \
|
||||||
"ac110002fe800000000000000042aafffe73bbfaff7e6aaaff7e6aaa000000400000000100000000" \
|
"ac110002fe800000000000000042aafffe73bbfaff7e6aaaff7e6aaa000000400000000100000000" \
|
||||||
"0000000088000600fe800000000000000042acfffe110002fe800000000000000042aafffe73bbfa" \
|
"0000000088000600fe800000000000000042acfffe110002fe800000000000000042aafffe73bbfa" \
|
||||||
|
@ -213,19 +214,71 @@ PACKET_IPFIX_TEMPLATE = "000a05205e8465fd0000001300000000000200400400000e0008000
|
||||||
"0000000088000600fe800000000000000042aafffe73bbfafe800000000000000042acfffe110002" \
|
"0000000088000600fe800000000000000042aafffe73bbfafe800000000000000042acfffe110002" \
|
||||||
"ff7e92aaff7e92aa0000004800000001000000000000000087000600fe800000000000000042acff" \
|
"ff7e92aaff7e92aa0000004800000001000000000000000087000600fe800000000000000042acff" \
|
||||||
"fe110002fe800000000000000042aafffe73bbfaff7e92aaff7e92aa000000400000000100000000" \
|
"fe110002fe800000000000000042aafffe73bbfaff7e92aaff7e92aa000000400000000100000000" \
|
||||||
"000000008800060008000044fde66f14e0f1960900000242ac110002fd41b7143f86000000000000" \
|
"000000008800060008000044fde66f14e0f196090000affeaffeaffefd41b7143f86000000000000" \
|
||||||
"00000001ff7ec2a0ff7ec2a00000004a000000010000000000000000d20100351100060004000054" \
|
"00000001ff7ec2a0ff7ec2a00000004a000000010000000000000000d20100351100060004000054" \
|
||||||
"ac1100027f000001ff7ed62eff7ed68700000036000000010000000000000000c496003511000400" \
|
"ac1100027f000001ff7ed62eff7ed68700000036000000010000000000000000c496003511000400" \
|
||||||
"7f000001ac110002ff7ed62eff7ed687000000760000000100000000000000000035c49611000400" \
|
"7f000001ac110002ff7ed62eff7ed687000000760000000100000000000000000035c49611000400" \
|
||||||
"08000044fde66f14e0f1960900000242ac110002fd41b7143f8600000000000000000001ff7ef359" \
|
"08000044fde66f14e0f196090000affeaffeaffefd41b7143f8600000000000000000001ff7ef359" \
|
||||||
"ff7ef3590000004a000000010000000000000000b1e700351100060004000054ac1100027f000001" \
|
"ff7ef3590000004a000000010000000000000000b1e700351100060004000054ac1100027f000001" \
|
||||||
"ff7f06e4ff7f06e800000036000000010000000000000000a8f90035110004007f000001ac110002" \
|
"ff7f06e4ff7f06e800000036000000010000000000000000a8f90035110004007f000001ac110002" \
|
||||||
"ff7f06e4ff7f06e8000000a60000000100000000000000000035a8f911000400"
|
"ff7f06e4ff7f06e8000000a60000000100000000000000000035a8f911000400"
|
||||||
|
|
||||||
# Example export for IPFIX with two data sets
|
# Example export for IPFIX with two data sets
|
||||||
PACKET_IPFIX = "000a00d05e8465fd00000016000000000801007cfe800000000000000042acfffe110002fde66f14" \
|
PACKET_IPFIX = "000a00d02d45a47000000016000000000801007cfe800000000000000042acfffe110002fde66f14" \
|
||||||
"e0f196090000000000000001ff7f0755ff7f07550000004800000001000000000000000087000600" \
|
"e0f196090000000000000001ff7f0755ff7f07550000004800000001000000000000000087000600" \
|
||||||
"fde66f14e0f196090000000000000001fe800000000000000042acfffe110002ff7f0755ff7f0755" \
|
"fdabcdef123456789000000000000001fe800000000000000042acfffe110002ff7f0755ff7f0755" \
|
||||||
"000000400000000100000000000000008800060008000044fde66f14e0f1960900000242ac110002" \
|
"000000400000000100000000000000008800060008000044fde66f14e0f196090000affeaffeaffe" \
|
||||||
"2a044e42020000000000000000000223ff7f06e9ff7f22d500000140000000040000000000000000" \
|
"2a044e42020000000000000000000223ff7f06e9ff7f22d500000140000000040000000000000000" \
|
||||||
"e54c01bb06020600"
|
"e54c01bb06020600"
|
||||||
|
|
||||||
|
PACKET_IPFIX_TEMPLATE_ETHER = "000a05002d45a4700000000d00000000" \
|
||||||
|
"000200500400001200080004000c000400160004001500040001000400020004000a0004000e0004" \
|
||||||
|
"00070002000b00020004000100060001003c000100050001003a0002003b00020038000600390006" \
|
||||||
|
"000200440401000f00080004000c000400160004001500040001000400020004000a0004000e0004" \
|
||||||
|
"00200002003c000100050001003a0002003b000200380006003900060002005008000012001b0010" \
|
||||||
|
"001c001000160004001500040001000400020004000a0004000e000400070002000b000200040001" \
|
||||||
|
"00060001003c000100050001003a0002003b00020038000600390006000200440801000f001b0010" \
|
||||||
|
"001c001000160004001500040001000400020004000a0004000e0004008b0002003c000100050001" \
|
||||||
|
"003a0002003b000200380006003900060003001e010000050001008f000400a00008013100040132" \
|
||||||
|
"0004013000020100001a00000009000000b0d80a558000000001000000000001040000747f000001" \
|
||||||
|
"ac110002e58b988be58b993e000015c70000000d000000000000000001bbe1a6061b040000000000" \
|
||||||
|
"123456affefeaffeaffeaffeac1100027f000001e58b988be58b993e0000074f0000001300000000" \
|
||||||
|
"00000000e1a601bb061f040000000000affeaffeaffe123456affefe0401006cac110002ac110001" \
|
||||||
|
"e58a7be3e58a83d3000000fc0000000300000000000000000800040000000000affeaffeaffe0242" \
|
||||||
|
"aa73bbfaac110001ac110002e58a7be3e58a83d3000000fc00000003000000000000000000000400" \
|
||||||
|
"00000000123456affefeaffeaffeaffe080102b0fde66f14e0f196090000affeaffeaffeff020000" \
|
||||||
|
"0000000000000001ff110001e58abcd9e58ad098000001b000000006000000000000000087000600" \
|
||||||
|
"00000000affeaffeaffe3333ff110001fde66f14e0f196090000affeaffeaffefde66f14e0f19609" \
|
||||||
|
"0000000000000001e58b1883e58b284e000002080000000500000000000000008000060000000000" \
|
||||||
|
"affeaffeaffe123456affefefdabcdef123456789000000000000001fde66f14e0f1960900000242" \
|
||||||
|
"ac110002e58b1883e58b284e0000020800000005000000000000000081000600000000000242aa73" \
|
||||||
|
"bbfaaffeaffeaffefe800000000000000042aafffe73bbfafde66f14e0f196090000affeaffeaffe" \
|
||||||
|
"e58b2caee58b2cae000000480000000100000000000000008700060000000000123456affefe0242" \
|
||||||
|
"ac110002fde66f14e0f196090000affeaffeaffefe800000000000000042aafffe73bbfae58b2cae" \
|
||||||
|
"e58b2cae000000400000000100000000000000008800060000000000affeaffeaffe123456affefe" \
|
||||||
|
"fe800000000000000042acfffe110002fe800000000000000042aafffe73bbfae58b40aee58b40ae" \
|
||||||
|
"000000480000000100000000000000008700060000000000affeaffeaffe123456affefefe800000" \
|
||||||
|
"000000000042aafffe73bbfafe800000000000000042acfffe110002e58b40aee58b40ae00000040" \
|
||||||
|
"0000000100000000000000008800060000000000123456affefeaffeaffeaffefe80000000000000" \
|
||||||
|
"0042aafffe73bbfafe800000000000000042acfffe110002e58b54aee58b54ae0000004800000001" \
|
||||||
|
"00000000000000008700060000000000123456affefeaffeaffeaffefe800000000000000042acff" \
|
||||||
|
"fe110002fe800000000000000042aafffe73bbfae58b54aee58b54ae000000400000000100000000" \
|
||||||
|
"000000008800060000000000affeaffeaffe123456affefe"
|
||||||
|
|
||||||
|
PACKET_IPFIX_ETHER = "000a02905e8b0aa90000001600000000" \
|
||||||
|
"08000054fde66f14e0f196090000affeaffeaffefd40abcdabcd00000000000000011111e58b84a4" \
|
||||||
|
"e58b84a40000004a000000010000000000000000d20100351100060000000000affeaffeaffe0242" \
|
||||||
|
"aa73bbfa04000074ac1100027f000001e58b9831e58b988a00000036000000010000000000000000" \
|
||||||
|
"c49600351100040000000000affeaffeaffe123456affefe7f000001ac110002e58b9831e58b988a" \
|
||||||
|
"000000760000000100000000000000000035c4961100040000000000123456affefeaffeaffeaffe" \
|
||||||
|
"08000054fde66f14e0f196090000affeaffeaffefd40abcdabcd00000000000000011111e58bb55c" \
|
||||||
|
"e58bb55c0000004a000000010000000000000000b1e700351100060000000000affeaffeaffe0242" \
|
||||||
|
"aa73bbfa04000074ac1100027f000001e58bc8e8e58bc8ec00000036000000010000000000000000" \
|
||||||
|
"a8f900351100040000000000affeaffeaffe123456affefe7f000001ac110002e58bc8e8e58bc8ec" \
|
||||||
|
"000000a60000000100000000000000000035a8f91100040000000000123456affefeaffeaffeaffe" \
|
||||||
|
"0801009cfe800000000000000042acfffe110002fdabcdef123456789000000000000001e58bc958" \
|
||||||
|
"e58bc958000000480000000100000000000000008700060000000000affeaffeaffe123456affefe" \
|
||||||
|
"fdabcdef123456789000000000000001fe800000000000000042acfffe110002e58bc958e58bc958" \
|
||||||
|
"000000400000000100000000000000008800060000000000123456affefeaffeaffeaffe08000054" \
|
||||||
|
"fde66f14e0f196090000affeaffeaffe2a044e42020000000000000000000223e58bc8ede58be4d8" \
|
||||||
|
"00000140000000040000000000000000e54c01bb0602060000000000affeaffeaffe123456affefe"
|
||||||
|
|
|
@ -11,13 +11,18 @@ Licensed under MIT License. See LICENSE.
|
||||||
import ipaddress
|
import ipaddress
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from tests.lib import send_recv_packets, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX
|
from tests.lib import send_recv_packets, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX, PACKET_IPFIX_ETHER, \
|
||||||
|
PACKET_IPFIX_TEMPLATE_ETHER
|
||||||
|
|
||||||
|
|
||||||
class TestFlowExportIPFIX(unittest.TestCase):
|
class TestFlowExportIPFIX(unittest.TestCase):
|
||||||
"""Test IPFIX packet parsing
|
|
||||||
"""
|
|
||||||
def test_recv_ipfix_packet(self):
|
def test_recv_ipfix_packet(self):
|
||||||
|
"""
|
||||||
|
Test general sending of raw and receiving and parsing of these packets.
|
||||||
|
If this test runs successfully, the sender thread has sent a raw bytes packet towards a locally
|
||||||
|
listening collector thread, and the collector has successfully received and parsed the packets.
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
# send packet without any template, must fail to parse (packets are queued)
|
# send packet without any template, must fail to parse (packets are queued)
|
||||||
pkts, _, _ = send_recv_packets([PACKET_IPFIX])
|
pkts, _, _ = send_recv_packets([PACKET_IPFIX])
|
||||||
self.assertEqual(len(pkts), 0) # no export is parsed due to missing template
|
self.assertEqual(len(pkts), 0) # no export is parsed due to missing template
|
||||||
|
@ -25,12 +30,31 @@ class TestFlowExportIPFIX(unittest.TestCase):
|
||||||
# send packet with 5 templates and 20 flows, should parse correctly since the templates are known
|
# send packet with 5 templates and 20 flows, should parse correctly since the templates are known
|
||||||
pkts, _, _ = send_recv_packets([PACKET_IPFIX_TEMPLATE])
|
pkts, _, _ = send_recv_packets([PACKET_IPFIX_TEMPLATE])
|
||||||
self.assertEqual(len(pkts), 1)
|
self.assertEqual(len(pkts), 1)
|
||||||
|
|
||||||
p = pkts[0]
|
p = pkts[0]
|
||||||
self.assertEqual(p.client[0], "127.0.0.1")
|
self.assertEqual(p.client[0], "127.0.0.1")
|
||||||
self.assertEqual(len(p.export.flows), 1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) # count flows
|
self.assertEqual(len(p.export.flows), 1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) # count flows
|
||||||
self.assertEqual(len(p.export.templates), 4 + 1) # count new templates
|
self.assertEqual(len(p.export.templates), 4 + 1) # count new templates
|
||||||
|
|
||||||
# Inspect contents of specific flows
|
# send template and multiple export packets
|
||||||
|
pkts, _, _ = send_recv_packets([PACKET_IPFIX, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX])
|
||||||
|
self.assertEqual(len(pkts), 3)
|
||||||
|
self.assertEqual(pkts[0].export.header.version, 10)
|
||||||
|
|
||||||
|
# check amount of flows across all packets
|
||||||
|
total_flows = 0
|
||||||
|
for packet in pkts:
|
||||||
|
total_flows += len(packet.export.flows)
|
||||||
|
self.assertEqual(total_flows, 2 + 1 + (1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) + 2 + 1)
|
||||||
|
|
||||||
|
def test_ipfix_contents(self):
|
||||||
|
"""
|
||||||
|
Inspect content of exported flows, eg. test the value of an option flow and the correct
|
||||||
|
parsing of IPv4 and IPv6 addresses.
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
p = send_recv_packets([PACKET_IPFIX_TEMPLATE])[0][0]
|
||||||
|
|
||||||
flow = p.export.flows[0]
|
flow = p.export.flows[0]
|
||||||
self.assertEqual(flow.meteringProcessId, 2649)
|
self.assertEqual(flow.meteringProcessId, 2649)
|
||||||
self.assertEqual(flow.selectorAlgorithm, 1)
|
self.assertEqual(flow.selectorAlgorithm, 1)
|
||||||
|
@ -43,20 +67,33 @@ class TestFlowExportIPFIX(unittest.TestCase):
|
||||||
self.assertEqual(flow.protocolIdentifier, 6) # TCP
|
self.assertEqual(flow.protocolIdentifier, 6) # TCP
|
||||||
self.assertEqual(flow.sourceTransportPort, 443)
|
self.assertEqual(flow.sourceTransportPort, 443)
|
||||||
self.assertEqual(flow.destinationTransportPort, 57766)
|
self.assertEqual(flow.destinationTransportPort, 57766)
|
||||||
|
self.assertEqual(flow.tcpControlBits, 0x1b)
|
||||||
|
|
||||||
flow = p.export.flows[17] # IPv6 flow
|
flow = p.export.flows[17] # IPv6 flow
|
||||||
self.assertEqual(flow.protocolIdentifier, 17) # UDP
|
self.assertEqual(flow.protocolIdentifier, 17) # UDP
|
||||||
self.assertEqual(flow.sourceIPv6Address, 337491164212692683663430561043420610562)
|
self.assertEqual(flow.sourceIPv6Address, 0xfde66f14e0f196090000affeaffeaffe)
|
||||||
self.assertEqual(ipaddress.ip_address(flow.sourceIPv6Address), # Docker ULA
|
self.assertEqual(ipaddress.ip_address(flow.sourceIPv6Address), # Docker ULA
|
||||||
ipaddress.ip_address("fde6:6f14:e0f1:9609:0:242:ac11:2"))
|
ipaddress.ip_address("fde6:6f14:e0f1:9609:0:affe:affe:affe"))
|
||||||
|
|
||||||
# send template and multiple export packets
|
def test_ipfix_contents_ether(self):
|
||||||
pkts, _, _ = send_recv_packets([PACKET_IPFIX, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX])
|
"""
|
||||||
self.assertEqual(len(pkts), 3)
|
IPFIX content tests based on exports with the softflowd "-T ether" flag, meaning that layer 2
|
||||||
self.assertEqual(pkts[0].export.header.version, 10)
|
is included in the export, like MAC addresses.
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
pkts, _, _ = send_recv_packets([PACKET_IPFIX_TEMPLATE_ETHER, PACKET_IPFIX_ETHER])
|
||||||
|
self.assertEqual(len(pkts), 2)
|
||||||
|
p = pkts[0]
|
||||||
|
|
||||||
# check amount of flows across all packets
|
# Inspect contents of specific flows
|
||||||
total_flows = 0
|
flow = p.export.flows[0]
|
||||||
for packet in pkts:
|
self.assertEqual(flow.meteringProcessId, 9)
|
||||||
total_flows += len(packet.export.flows)
|
self.assertEqual(flow.selectorAlgorithm, 1)
|
||||||
self.assertEqual(total_flows, 2 + 1 + (1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) + 2 + 1)
|
self.assertEqual(flow.systemInitTimeMilliseconds, 759538800000)
|
||||||
|
|
||||||
|
flow = p.export.flows[1]
|
||||||
|
self.assertEqual(flow.destinationIPv4Address, 2886795266)
|
||||||
|
self.assertTrue(hasattr(flow, "sourceMacAddress"))
|
||||||
|
self.assertTrue(hasattr(flow, "postDestinationMacAddress"))
|
||||||
|
self.assertEqual(flow.sourceMacAddress, 0x123456affefe)
|
||||||
|
self.assertEqual(flow.postDestinationMacAddress, 0xaffeaffeaffe)
|
||||||
|
|
|
@ -12,8 +12,9 @@ import ipaddress
|
||||||
import random
|
import random
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from tests.lib import send_recv_packets, NUM_PACKETS, PACKET_INVALID, PACKET_V1, PACKET_V5, PACKET_V9_TEMPLATE, \
|
from tests.lib import send_recv_packets, NUM_PACKETS, \
|
||||||
PACKET_V9_TEMPLATE_MIXED, PACKETS_V9
|
PACKET_INVALID, PACKET_V1, PACKET_V5, \
|
||||||
|
PACKET_V9_TEMPLATE, PACKET_V9_TEMPLATE_MIXED, PACKETS_V9
|
||||||
|
|
||||||
|
|
||||||
class TestFlowExportNetflow(unittest.TestCase):
|
class TestFlowExportNetflow(unittest.TestCase):
|
||||||
|
@ -150,7 +151,3 @@ class TestFlowExportNetflow(unittest.TestCase):
|
||||||
for packet in pkts:
|
for packet in pkts:
|
||||||
total_flows += len(packet.export.flows)
|
total_flows += len(packet.export.flows)
|
||||||
self.assertEqual(total_flows, 8 + 12 + 12 + 12)
|
self.assertEqual(total_flows, 8 + 12 + 12 + 12)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
unittest.main()
|
|
||||||
|
|
Loading…
Reference in a new issue