commit
a86fe7c731
260
netflow/v9.py
260
netflow/v9.py
|
@ -15,8 +15,11 @@ Licensed under MIT License. See LICENSE.
|
||||||
import ipaddress
|
import ipaddress
|
||||||
import struct
|
import struct
|
||||||
|
|
||||||
__all__ = ["V9DataFlowSet", "V9DataRecord", "V9ExportPacket", "V9Header", "V9TemplateField", "V9OptionsTemplateFlowSet"
|
from .ipfix import IPFIXFieldTypes, IPFIXDataTypes
|
||||||
"V9TemplateFlowSet", "V9TemplateNotRecognized", "V9TemplateRecord"]
|
|
||||||
|
__all__ = ["V9DataFlowSet", "V9DataRecord", "V9ExportPacket", "V9Header", "V9TemplateField",
|
||||||
|
"V9TemplateFlowSet", "V9TemplateNotRecognized", "V9TemplateRecord",
|
||||||
|
"V9OptionsTemplateFlowSet", "V9OptionsTemplateRecord", "V9OptionsDataRecord"]
|
||||||
|
|
||||||
V9_FIELD_TYPES = {
|
V9_FIELD_TYPES = {
|
||||||
0: 'UNKNOWN_FIELD_TYPE', # fallback for unknown field types
|
0: 'UNKNOWN_FIELD_TYPE', # fallback for unknown field types
|
||||||
|
@ -155,6 +158,14 @@ V9_FIELD_TYPES = {
|
||||||
56702: 'PANOS_USERID'
|
56702: 'PANOS_USERID'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
V9_SCOPE_TYPES = {
|
||||||
|
1: "System",
|
||||||
|
2: "Interface",
|
||||||
|
3: "Line Card",
|
||||||
|
4: "Cache",
|
||||||
|
5: "Template"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class V9TemplateNotRecognized(KeyError):
|
class V9TemplateNotRecognized(KeyError):
|
||||||
pass
|
pass
|
||||||
|
@ -181,7 +192,7 @@ class V9DataFlowSet:
|
||||||
DataFlowSet and must not be zero.
|
DataFlowSet and must not be zero.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, data, templates):
|
def __init__(self, data, template):
|
||||||
pack = struct.unpack('!HH', data[:4])
|
pack = struct.unpack('!HH', data[:4])
|
||||||
|
|
||||||
self.template_id = pack[0] # flowset_id is reference to a template_id
|
self.template_id = pack[0] # flowset_id is reference to a template_id
|
||||||
|
@ -190,11 +201,6 @@ class V9DataFlowSet:
|
||||||
|
|
||||||
offset = 4
|
offset = 4
|
||||||
|
|
||||||
if self.template_id not in templates:
|
|
||||||
raise V9TemplateNotRecognized
|
|
||||||
|
|
||||||
template = templates[self.template_id]
|
|
||||||
|
|
||||||
# As the field lengths are variable V9 has padding to next 32 Bit
|
# As the field lengths are variable V9 has padding to next 32 Bit
|
||||||
padding_size = 4 - (self.length % 4) # 4 Byte
|
padding_size = 4 - (self.length % 4) # 4 Byte
|
||||||
|
|
||||||
|
@ -252,7 +258,7 @@ class V9TemplateRecord:
|
||||||
"""A template record contained in a TemplateFlowSet.
|
"""A template record contained in a TemplateFlowSet.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, template_id, field_count, fields):
|
def __init__(self, template_id, field_count, fields: list):
|
||||||
self.template_id = template_id
|
self.template_id = template_id
|
||||||
self.field_count = field_count
|
self.field_count = field_count
|
||||||
self.fields = fields
|
self.fields = fields
|
||||||
|
@ -263,15 +269,141 @@ class V9TemplateRecord:
|
||||||
' '.join([V9_FIELD_TYPES[field.field_type] for field in self.fields]))
|
' '.join([V9_FIELD_TYPES[field.field_type] for field in self.fields]))
|
||||||
|
|
||||||
|
|
||||||
class V9OptionsTemplateFlowSet:
|
class V9OptionsDataRecord:
|
||||||
"""An options template flowset. Always uses flowset ID 1.
|
def __init__(self):
|
||||||
TODO: not handled at the moment, only stub implementation
|
self.scopes = {}
|
||||||
|
self.data = {}
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return "<V9OptionsDataRecord with scopes {} and data {}>".format(self.scopes.keys(), self.data.keys())
|
||||||
|
|
||||||
|
|
||||||
|
class V9OptionsTemplateRecord:
|
||||||
|
"""An options template record contained in an options template flowset.
|
||||||
"""
|
"""
|
||||||
def __init__(self, data):
|
|
||||||
pack = struct.unpack('!HHH', data[:6])
|
def __init__(self, template_id, scope_fields: dict, option_fields: dict):
|
||||||
self.flowset_id = pack[0]
|
self.template_id = template_id
|
||||||
|
self.scope_fields = scope_fields
|
||||||
|
self.option_fields = option_fields
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return "<V9OptionsTemplateRecord with scope fields {} and option fields {}>".format(
|
||||||
|
self.scope_fields.keys(), self.option_fields.keys())
|
||||||
|
|
||||||
|
|
||||||
|
class V9OptionsTemplateFlowSet:
|
||||||
|
"""An options template flowset.
|
||||||
|
|
||||||
|
> Each Options Template FlowSet MAY contain multiple Options Template Records.
|
||||||
|
|
||||||
|
Scope field types range from 1 to 5:
|
||||||
|
1 System
|
||||||
|
2 Interface
|
||||||
|
3 Line Card
|
||||||
|
4 Cache
|
||||||
|
5 Template
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, data: bytes):
|
||||||
|
pack = struct.unpack('!HH', data[:4])
|
||||||
|
self.flowset_id = pack[0] # always 1
|
||||||
|
self.flowset_length = pack[1] # length of this flowset
|
||||||
|
self.templates = {}
|
||||||
|
|
||||||
|
offset = 4
|
||||||
|
|
||||||
|
while offset < self.flowset_length:
|
||||||
|
pack = struct.unpack("!HHH", data[offset:offset + 6]) # options template header
|
||||||
|
template_id = pack[0] # value above 255
|
||||||
|
option_scope_length = pack[1]
|
||||||
|
options_length = pack[2]
|
||||||
|
|
||||||
|
offset += 6
|
||||||
|
|
||||||
|
# Fetch all scope fields (most probably only one field)
|
||||||
|
scopes = {} # Holds "type: length" key-value pairs
|
||||||
|
|
||||||
|
if option_scope_length % 4 != 0 or options_length % 4 != 0:
|
||||||
|
raise ValueError(option_scope_length, options_length)
|
||||||
|
|
||||||
|
for scope_counter in range(option_scope_length // 4): # example: option_scope_length = 4 means one scope
|
||||||
|
pack = struct.unpack("!HH", data[offset:offset + 4])
|
||||||
|
scope_field_type = pack[0] # values range from 1 to 5
|
||||||
|
scope_field_length = pack[1]
|
||||||
|
scopes[scope_field_type] = scope_field_length
|
||||||
|
offset += 4
|
||||||
|
|
||||||
|
# Fetch all option fields
|
||||||
|
options = {} # same
|
||||||
|
for option_counter in range(options_length // 4): # now counting the options
|
||||||
|
pack = struct.unpack("!HH", data[offset:offset + 4])
|
||||||
|
option_field_type = pack[0]
|
||||||
|
option_field_length = pack[1]
|
||||||
|
options[option_field_type] = option_field_length
|
||||||
|
offset += 4
|
||||||
|
|
||||||
|
optionstemplate = V9OptionsTemplateRecord(template_id, scopes, options)
|
||||||
|
|
||||||
|
self.templates[template_id] = optionstemplate
|
||||||
|
|
||||||
|
# handle padding and add offset if needed
|
||||||
|
if offset % 4 == 2:
|
||||||
|
offset += 2
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return "<V9OptionsTemplateFlowSet with {} templates: {}>".format(len(self.templates), self.templates.keys())
|
||||||
|
|
||||||
|
|
||||||
|
class V9OptionsDataFlowset:
|
||||||
|
"""An options data flowset with option data records
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, data: bytes, template: V9OptionsTemplateRecord):
|
||||||
|
pack = struct.unpack('!HH', data[:4])
|
||||||
|
|
||||||
|
self.template_id = pack[0]
|
||||||
self.length = pack[1]
|
self.length = pack[1]
|
||||||
self.template_id = pack[2]
|
self.option_data_records = []
|
||||||
|
|
||||||
|
offset = 4
|
||||||
|
|
||||||
|
while offset < self.length:
|
||||||
|
new_options_record = V9OptionsDataRecord()
|
||||||
|
|
||||||
|
for scope_type, length in template.scope_fields.items():
|
||||||
|
type_name = V9_SCOPE_TYPES.get(scope_type, scope_type) # Either name, or unknown int
|
||||||
|
value = int.from_bytes(data[offset:offset+length], 'big') # TODO: is this always integer?
|
||||||
|
new_options_record.scopes[type_name] = value
|
||||||
|
offset += length
|
||||||
|
|
||||||
|
for field_type, length in template.option_fields.items():
|
||||||
|
type_name = V9_FIELD_TYPES.get(field_type, None)
|
||||||
|
is_bytes = False
|
||||||
|
|
||||||
|
if not type_name: # Cisco refers to the IANA IPFIX table for types >256...
|
||||||
|
iana_type = IPFIXFieldTypes.by_id(field_type) # try to get from IPFIX types
|
||||||
|
if iana_type:
|
||||||
|
type_name = iana_type.name
|
||||||
|
is_bytes = IPFIXDataTypes.is_bytes(iana_type)
|
||||||
|
|
||||||
|
if not type_name:
|
||||||
|
raise ValueError
|
||||||
|
|
||||||
|
value = None
|
||||||
|
if is_bytes:
|
||||||
|
value = data[offset:offset+length]
|
||||||
|
else:
|
||||||
|
value = int.from_bytes(data[offset:offset+length], 'big')
|
||||||
|
|
||||||
|
new_options_record.data[type_name] = value
|
||||||
|
|
||||||
|
offset += length
|
||||||
|
|
||||||
|
self.option_data_records.append(new_options_record)
|
||||||
|
|
||||||
|
if offset % 4 == 2:
|
||||||
|
offset += 2
|
||||||
|
|
||||||
|
|
||||||
class V9TemplateFlowSet:
|
class V9TemplateFlowSet:
|
||||||
|
@ -283,7 +415,7 @@ class V9TemplateFlowSet:
|
||||||
|
|
||||||
def __init__(self, data):
|
def __init__(self, data):
|
||||||
pack = struct.unpack('!HH', data[:4])
|
pack = struct.unpack('!HH', data[:4])
|
||||||
self.flowset_id = pack[0]
|
self.flowset_id = pack[0] # always 0
|
||||||
self.length = pack[1] # total length including this header in bytes
|
self.length = pack[1] # total length including this header in bytes
|
||||||
self.templates = {}
|
self.templates = {}
|
||||||
|
|
||||||
|
@ -339,55 +471,90 @@ class V9Header:
|
||||||
|
|
||||||
class V9ExportPacket:
|
class V9ExportPacket:
|
||||||
"""The flow record holds the header and all template and data flowsets.
|
"""The flow record holds the header and all template and data flowsets.
|
||||||
|
|
||||||
|
TODO: refactor into two loops: first get all contained flowsets and examine template
|
||||||
|
flowsets first. Then data flowsets.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, data, templates):
|
def __init__(self, data: bytes, templates: dict):
|
||||||
self.header = V9Header(data)
|
self.header = V9Header(data)
|
||||||
self._templates = templates
|
self._templates = templates
|
||||||
self._new_templates = False
|
self._new_templates = False
|
||||||
self._flows = []
|
self._flows = []
|
||||||
|
self._options = []
|
||||||
|
|
||||||
offset = self.header.length
|
offset = self.header.length
|
||||||
skipped_flowsets_offsets = []
|
skipped_flowsets_offsets = []
|
||||||
while offset != len(data):
|
|
||||||
flowset_id = struct.unpack('!H', data[offset:offset + 2])[0]
|
|
||||||
|
|
||||||
|
while offset != len(data):
|
||||||
|
pack = struct.unpack('!HH', data[offset:offset + 4])
|
||||||
|
flowset_id = pack[0] # = template id
|
||||||
|
flowset_length = pack[1]
|
||||||
|
|
||||||
|
# Data template flowsets
|
||||||
if flowset_id == 0: # TemplateFlowSet always have id 0
|
if flowset_id == 0: # TemplateFlowSet always have id 0
|
||||||
tfs = V9TemplateFlowSet(data[offset:])
|
tfs = V9TemplateFlowSet(data[offset:])
|
||||||
|
|
||||||
# Check for any new/changed templates
|
|
||||||
if not self._new_templates:
|
|
||||||
for id_, template in tfs.templates.items():
|
|
||||||
if id_ not in self._templates or self._templates[id_] != template:
|
|
||||||
self._new_templates = True
|
|
||||||
break
|
|
||||||
|
|
||||||
# Update the templates with the provided templates, even if they are the same
|
# Update the templates with the provided templates, even if they are the same
|
||||||
self._templates.update(tfs.templates)
|
for id_, template in tfs.templates.items():
|
||||||
|
if id_ not in self._templates:
|
||||||
|
self._new_templates = True
|
||||||
|
self._templates[id_] = template
|
||||||
offset += tfs.length
|
offset += tfs.length
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Option template flowsets
|
||||||
elif flowset_id == 1: # Option templates always use ID 1
|
elif flowset_id == 1: # Option templates always use ID 1
|
||||||
# TODO: Options templates are ignored, to prevent template ID collision
|
|
||||||
# (if a collision can occur is not yet tested)
|
|
||||||
otfs = V9OptionsTemplateFlowSet(data[offset:])
|
otfs = V9OptionsTemplateFlowSet(data[offset:])
|
||||||
offset += otfs.length
|
for id_, template in otfs.templates.items():
|
||||||
|
if id_ not in self._templates:
|
||||||
|
self._new_templates = True
|
||||||
|
self._templates[id_] = template
|
||||||
|
offset += otfs.flowset_length
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Data / option flowsets
|
||||||
|
# First, check if template is known
|
||||||
|
if flowset_id not in self._templates:
|
||||||
|
# Could not be parsed, continue to check for templates
|
||||||
|
skipped_flowsets_offsets.append(offset)
|
||||||
|
offset += flowset_length
|
||||||
|
continue
|
||||||
|
|
||||||
|
matched_template = self._templates[flowset_id]
|
||||||
|
|
||||||
|
if isinstance(matched_template, V9TemplateRecord):
|
||||||
|
dfs = V9DataFlowSet(data[offset:], matched_template)
|
||||||
|
self._flows += dfs.flows
|
||||||
|
offset += dfs.length
|
||||||
|
|
||||||
|
elif isinstance(matched_template, V9OptionsTemplateRecord):
|
||||||
|
odfs = V9OptionsDataFlowset(data[offset:], matched_template)
|
||||||
|
self._options += odfs.option_data_records
|
||||||
|
offset += odfs.length
|
||||||
|
|
||||||
else:
|
else:
|
||||||
try:
|
raise NotImplementedError
|
||||||
dfs = V9DataFlowSet(data[offset:], self._templates)
|
|
||||||
self._flows += dfs.flows
|
|
||||||
offset += dfs.length
|
|
||||||
except V9TemplateNotRecognized:
|
|
||||||
# Could not be parsed, continue to check for templates
|
|
||||||
length = struct.unpack("!H", data[offset + 2:offset + 4])[0]
|
|
||||||
skipped_flowsets_offsets.append(offset)
|
|
||||||
offset += length
|
|
||||||
|
|
||||||
|
# In the same export packet, re-try flowsets with previously unknown templates.
|
||||||
|
# Might happen, if an export packet first contains data flowsets, and template flowsets after
|
||||||
if skipped_flowsets_offsets and self._new_templates:
|
if skipped_flowsets_offsets and self._new_templates:
|
||||||
# Process flowsets in the data slice which occured before the template sets
|
# Process flowsets in the data slice which occured before the template sets
|
||||||
|
# Handling of offset increases is not needed here
|
||||||
for offset in skipped_flowsets_offsets:
|
for offset in skipped_flowsets_offsets:
|
||||||
dfs = V9DataFlowSet(data[offset:], self._templates)
|
pack = struct.unpack('!H', data[offset:offset + 2])
|
||||||
self._flows += dfs.flows
|
flowset_id = pack[0]
|
||||||
|
|
||||||
|
if flowset_id not in self._templates:
|
||||||
|
raise V9TemplateNotRecognized
|
||||||
|
|
||||||
|
matched_template = self._templates[flowset_id]
|
||||||
|
if isinstance(matched_template, V9TemplateRecord):
|
||||||
|
dfs = V9DataFlowSet(data[offset:], matched_template)
|
||||||
|
self._flows += dfs.flows
|
||||||
|
elif isinstance(matched_template, V9OptionsTemplateRecord):
|
||||||
|
odfs = V9OptionsDataFlowset(data[offset:], matched_template)
|
||||||
|
self._options += odfs.option_data_records
|
||||||
|
|
||||||
elif skipped_flowsets_offsets:
|
elif skipped_flowsets_offsets:
|
||||||
raise V9TemplateNotRecognized
|
raise V9TemplateNotRecognized
|
||||||
|
|
||||||
|
@ -403,7 +570,10 @@ class V9ExportPacket:
|
||||||
def templates(self):
|
def templates(self):
|
||||||
return self._templates
|
return self._templates
|
||||||
|
|
||||||
|
@property
|
||||||
|
def options(self):
|
||||||
|
return self._options
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
s = " and new template(s)" if self.contains_new_templates else ""
|
s = " and new template(s)" if self.contains_new_templates else ""
|
||||||
return "<ExportPacket v{} with {} records{}>".format(
|
return "<V9ExportPacket with {} records{}>".format(self.header.count, s)
|
||||||
self.header.version, self.header.count, s)
|
|
||||||
|
|
Loading…
Reference in a new issue