Merge branch 'grafolean-fix/performance-improvements'
Bump version to v0.11.2 Closes #38 Closes #39
This commit is contained in:
commit
ff24da9743
|
@ -14,6 +14,7 @@ Licensed under MIT License. See LICENSE.
|
||||||
|
|
||||||
import ipaddress
|
import ipaddress
|
||||||
import struct
|
import struct
|
||||||
|
import sys
|
||||||
|
|
||||||
from .ipfix import IPFIXFieldTypes, IPFIXDataTypes
|
from .ipfix import IPFIXFieldTypes, IPFIXDataTypes
|
||||||
|
|
||||||
|
@ -21,6 +22,8 @@ __all__ = ["V9DataFlowSet", "V9DataRecord", "V9ExportPacket", "V9Header", "V9Tem
|
||||||
"V9TemplateFlowSet", "V9TemplateNotRecognized", "V9TemplateRecord",
|
"V9TemplateFlowSet", "V9TemplateNotRecognized", "V9TemplateRecord",
|
||||||
"V9OptionsTemplateFlowSet", "V9OptionsTemplateRecord", "V9OptionsDataRecord"]
|
"V9OptionsTemplateFlowSet", "V9OptionsTemplateRecord", "V9OptionsDataRecord"]
|
||||||
|
|
||||||
|
V9_FIELD_TYPES_CONTAINING_IP = [8, 12, 15, 18, 27, 28, 62, 63]
|
||||||
|
|
||||||
V9_FIELD_TYPES = {
|
V9_FIELD_TYPES = {
|
||||||
0: 'UNKNOWN_FIELD_TYPE', # fallback for unknown field types
|
0: 'UNKNOWN_FIELD_TYPE', # fallback for unknown field types
|
||||||
|
|
||||||
|
@ -204,31 +207,54 @@ class V9DataFlowSet:
|
||||||
# As the field lengths are variable V9 has padding to next 32 Bit
|
# As the field lengths are variable V9 has padding to next 32 Bit
|
||||||
padding_size = 4 - (self.length % 4) # 4 Byte
|
padding_size = 4 - (self.length % 4) # 4 Byte
|
||||||
|
|
||||||
while offset <= (self.length - padding_size):
|
# For performance reasons, we use struct.unpack to get individual values. Here
|
||||||
new_record = V9DataRecord()
|
# we prepare the format string for parsing it. The format string is based on the template fields and their
|
||||||
|
# lengths. The string can then be re-used for every data record in the data stream
|
||||||
|
struct_format = '!'
|
||||||
|
struct_len = 0
|
||||||
|
for field in template.fields:
|
||||||
|
# The length of the value byte slice is defined in the template
|
||||||
|
flen = field.field_length
|
||||||
|
if flen == 4:
|
||||||
|
struct_format += 'L'
|
||||||
|
elif flen == 2:
|
||||||
|
struct_format += 'H'
|
||||||
|
elif flen == 1:
|
||||||
|
struct_format += 'B'
|
||||||
|
else:
|
||||||
|
struct_format += '%ds' % flen
|
||||||
|
struct_len += flen
|
||||||
|
|
||||||
for field in template.fields:
|
while offset <= (self.length - padding_size):
|
||||||
|
# Here we actually unpack the values, the struct format string is used in every data record
|
||||||
|
# iteration, until the final offset reaches the end of the whole data stream
|
||||||
|
unpacked_values = struct.unpack(struct_format, data[offset:offset + struct_len])
|
||||||
|
|
||||||
|
new_record = V9DataRecord()
|
||||||
|
for field, value in zip(template.fields, unpacked_values):
|
||||||
flen = field.field_length
|
flen = field.field_length
|
||||||
fkey = V9_FIELD_TYPES[field.field_type]
|
fkey = V9_FIELD_TYPES[field.field_type]
|
||||||
|
|
||||||
# The length of the value byte slice is defined in the template
|
|
||||||
dataslice = data[offset:offset + flen]
|
|
||||||
|
|
||||||
# Better solution than struct.unpack with variable field length
|
|
||||||
fdata = 0
|
|
||||||
for idx, byte in enumerate(reversed(bytearray(dataslice))):
|
|
||||||
fdata += byte << (idx * 8)
|
|
||||||
|
|
||||||
# Special handling of IP addresses to convert integers to strings to not lose precision in dump
|
# Special handling of IP addresses to convert integers to strings to not lose precision in dump
|
||||||
# TODO: might only be needed for IPv6
|
# TODO: might only be needed for IPv6
|
||||||
if fkey in ["IPV4_SRC_ADDR", "IPV4_DST_ADDR", "IPV6_SRC_ADDR", "IPV6_DST_ADDR"]:
|
if field.field_type in V9_FIELD_TYPES_CONTAINING_IP:
|
||||||
try:
|
try:
|
||||||
ip = ipaddress.ip_address(fdata)
|
ip = ipaddress.ip_address(value)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
print("IP address could not be parsed: {}".format(fdata))
|
print("IP address could not be parsed: {}".format(repr(value)))
|
||||||
continue
|
continue
|
||||||
new_record.data[fkey] = ip.compressed
|
new_record.data[fkey] = ip.compressed
|
||||||
|
elif flen in (1, 2, 4):
|
||||||
|
# These values are already converted to numbers by struct.unpack:
|
||||||
|
new_record.data[fkey] = value
|
||||||
else:
|
else:
|
||||||
|
# Caveat: this code assumes little-endian system (like x86)
|
||||||
|
if sys.byteorder != "little":
|
||||||
|
print("v9.py uses bit shifting for little endianness. Your processor is not little endian")
|
||||||
|
|
||||||
|
fdata = 0
|
||||||
|
for idx, byte in enumerate(reversed(bytearray(value))):
|
||||||
|
fdata += byte << (idx * 8)
|
||||||
new_record.data[fkey] = fdata
|
new_record.data[fkey] = fdata
|
||||||
|
|
||||||
offset += flen
|
offset += flen
|
||||||
|
@ -373,7 +399,7 @@ class V9OptionsDataFlowset:
|
||||||
|
|
||||||
for scope_type, length in template.scope_fields.items():
|
for scope_type, length in template.scope_fields.items():
|
||||||
type_name = V9_SCOPE_TYPES.get(scope_type, scope_type) # Either name, or unknown int
|
type_name = V9_SCOPE_TYPES.get(scope_type, scope_type) # Either name, or unknown int
|
||||||
value = int.from_bytes(data[offset:offset+length], 'big') # TODO: is this always integer?
|
value = int.from_bytes(data[offset:offset + length], 'big') # TODO: is this always integer?
|
||||||
new_options_record.scopes[type_name] = value
|
new_options_record.scopes[type_name] = value
|
||||||
offset += length
|
offset += length
|
||||||
|
|
||||||
|
@ -392,9 +418,9 @@ class V9OptionsDataFlowset:
|
||||||
|
|
||||||
value = None
|
value = None
|
||||||
if is_bytes:
|
if is_bytes:
|
||||||
value = data[offset:offset+length]
|
value = data[offset:offset + length]
|
||||||
else:
|
else:
|
||||||
value = int.from_bytes(data[offset:offset+length], 'big')
|
value = int.from_bytes(data[offset:offset + length], 'big')
|
||||||
|
|
||||||
new_options_record.data[type_name] = value
|
new_options_record.data[type_name] = value
|
||||||
|
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -7,7 +7,7 @@ with open("README.md", "r") as fh:
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
name='netflow',
|
name='netflow',
|
||||||
version='0.11.1',
|
version='0.11.2',
|
||||||
description='NetFlow v1, v5, v9 and IPFIX tool suite implemented in Python 3',
|
description='NetFlow v1, v5, v9 and IPFIX tool suite implemented in Python 3',
|
||||||
long_description=long_description,
|
long_description=long_description,
|
||||||
long_description_content_type='text/markdown',
|
long_description_content_type='text/markdown',
|
||||||
|
|
Loading…
Reference in a new issue