cookie
GitHub
commit
a86fe7c731
260
netflow/v9.py
260
netflow/v9.py
|
|
@ -15,8 +15,11 @@ Licensed under MIT License. See LICENSE.
|
|||
import ipaddress
|
||||
import struct
|
||||
|
||||
__all__ = ["V9DataFlowSet", "V9DataRecord", "V9ExportPacket", "V9Header", "V9TemplateField", "V9OptionsTemplateFlowSet"
|
||||
"V9TemplateFlowSet", "V9TemplateNotRecognized", "V9TemplateRecord"]
|
||||
from .ipfix import IPFIXFieldTypes, IPFIXDataTypes
|
||||
|
||||
__all__ = ["V9DataFlowSet", "V9DataRecord", "V9ExportPacket", "V9Header", "V9TemplateField",
|
||||
"V9TemplateFlowSet", "V9TemplateNotRecognized", "V9TemplateRecord",
|
||||
"V9OptionsTemplateFlowSet", "V9OptionsTemplateRecord", "V9OptionsDataRecord"]
|
||||
|
||||
V9_FIELD_TYPES = {
|
||||
0: 'UNKNOWN_FIELD_TYPE', # fallback for unknown field types
|
||||
|
|
@ -155,6 +158,14 @@ V9_FIELD_TYPES = {
|
|||
56702: 'PANOS_USERID'
|
||||
}
|
||||
|
||||
V9_SCOPE_TYPES = {
|
||||
1: "System",
|
||||
2: "Interface",
|
||||
3: "Line Card",
|
||||
4: "Cache",
|
||||
5: "Template"
|
||||
}
|
||||
|
||||
|
||||
class V9TemplateNotRecognized(KeyError):
|
||||
pass
|
||||
|
|
@ -181,7 +192,7 @@ class V9DataFlowSet:
|
|||
DataFlowSet and must not be zero.
|
||||
"""
|
||||
|
||||
def __init__(self, data, templates):
|
||||
def __init__(self, data, template):
|
||||
pack = struct.unpack('!HH', data[:4])
|
||||
|
||||
self.template_id = pack[0] # flowset_id is reference to a template_id
|
||||
|
|
@ -190,11 +201,6 @@ class V9DataFlowSet:
|
|||
|
||||
offset = 4
|
||||
|
||||
if self.template_id not in templates:
|
||||
raise V9TemplateNotRecognized
|
||||
|
||||
template = templates[self.template_id]
|
||||
|
||||
# As the field lengths are variable V9 has padding to next 32 Bit
|
||||
padding_size = 4 - (self.length % 4) # 4 Byte
|
||||
|
||||
|
|
@ -252,7 +258,7 @@ class V9TemplateRecord:
|
|||
"""A template record contained in a TemplateFlowSet.
|
||||
"""
|
||||
|
||||
def __init__(self, template_id, field_count, fields):
|
||||
def __init__(self, template_id, field_count, fields: list):
|
||||
self.template_id = template_id
|
||||
self.field_count = field_count
|
||||
self.fields = fields
|
||||
|
|
@ -263,15 +269,141 @@ class V9TemplateRecord:
|
|||
' '.join([V9_FIELD_TYPES[field.field_type] for field in self.fields]))
|
||||
|
||||
|
||||
class V9OptionsTemplateFlowSet:
|
||||
"""An options template flowset. Always uses flowset ID 1.
|
||||
TODO: not handled at the moment, only stub implementation
|
||||
class V9OptionsDataRecord:
|
||||
def __init__(self):
|
||||
self.scopes = {}
|
||||
self.data = {}
|
||||
|
||||
def __repr__(self):
|
||||
return "<V9OptionsDataRecord with scopes {} and data {}>".format(self.scopes.keys(), self.data.keys())
|
||||
|
||||
|
||||
class V9OptionsTemplateRecord:
|
||||
"""An options template record contained in an options template flowset.
|
||||
"""
|
||||
def __init__(self, data):
|
||||
pack = struct.unpack('!HHH', data[:6])
|
||||
self.flowset_id = pack[0]
|
||||
|
||||
def __init__(self, template_id, scope_fields: dict, option_fields: dict):
|
||||
self.template_id = template_id
|
||||
self.scope_fields = scope_fields
|
||||
self.option_fields = option_fields
|
||||
|
||||
def __repr__(self):
|
||||
return "<V9OptionsTemplateRecord with scope fields {} and option fields {}>".format(
|
||||
self.scope_fields.keys(), self.option_fields.keys())
|
||||
|
||||
|
||||
class V9OptionsTemplateFlowSet:
|
||||
"""An options template flowset.
|
||||
|
||||
> Each Options Template FlowSet MAY contain multiple Options Template Records.
|
||||
|
||||
Scope field types range from 1 to 5:
|
||||
1 System
|
||||
2 Interface
|
||||
3 Line Card
|
||||
4 Cache
|
||||
5 Template
|
||||
"""
|
||||
|
||||
def __init__(self, data: bytes):
|
||||
pack = struct.unpack('!HH', data[:4])
|
||||
self.flowset_id = pack[0] # always 1
|
||||
self.flowset_length = pack[1] # length of this flowset
|
||||
self.templates = {}
|
||||
|
||||
offset = 4
|
||||
|
||||
while offset < self.flowset_length:
|
||||
pack = struct.unpack("!HHH", data[offset:offset + 6]) # options template header
|
||||
template_id = pack[0] # value above 255
|
||||
option_scope_length = pack[1]
|
||||
options_length = pack[2]
|
||||
|
||||
offset += 6
|
||||
|
||||
# Fetch all scope fields (most probably only one field)
|
||||
scopes = {} # Holds "type: length" key-value pairs
|
||||
|
||||
if option_scope_length % 4 != 0 or options_length % 4 != 0:
|
||||
raise ValueError(option_scope_length, options_length)
|
||||
|
||||
for scope_counter in range(option_scope_length // 4): # example: option_scope_length = 4 means one scope
|
||||
pack = struct.unpack("!HH", data[offset:offset + 4])
|
||||
scope_field_type = pack[0] # values range from 1 to 5
|
||||
scope_field_length = pack[1]
|
||||
scopes[scope_field_type] = scope_field_length
|
||||
offset += 4
|
||||
|
||||
# Fetch all option fields
|
||||
options = {} # same
|
||||
for option_counter in range(options_length // 4): # now counting the options
|
||||
pack = struct.unpack("!HH", data[offset:offset + 4])
|
||||
option_field_type = pack[0]
|
||||
option_field_length = pack[1]
|
||||
options[option_field_type] = option_field_length
|
||||
offset += 4
|
||||
|
||||
optionstemplate = V9OptionsTemplateRecord(template_id, scopes, options)
|
||||
|
||||
self.templates[template_id] = optionstemplate
|
||||
|
||||
# handle padding and add offset if needed
|
||||
if offset % 4 == 2:
|
||||
offset += 2
|
||||
|
||||
def __repr__(self):
|
||||
return "<V9OptionsTemplateFlowSet with {} templates: {}>".format(len(self.templates), self.templates.keys())
|
||||
|
||||
|
||||
class V9OptionsDataFlowset:
|
||||
"""An options data flowset with option data records
|
||||
"""
|
||||
|
||||
def __init__(self, data: bytes, template: V9OptionsTemplateRecord):
|
||||
pack = struct.unpack('!HH', data[:4])
|
||||
|
||||
self.template_id = pack[0]
|
||||
self.length = pack[1]
|
||||
self.template_id = pack[2]
|
||||
self.option_data_records = []
|
||||
|
||||
offset = 4
|
||||
|
||||
while offset < self.length:
|
||||
new_options_record = V9OptionsDataRecord()
|
||||
|
||||
for scope_type, length in template.scope_fields.items():
|
||||
type_name = V9_SCOPE_TYPES.get(scope_type, scope_type) # Either name, or unknown int
|
||||
value = int.from_bytes(data[offset:offset+length], 'big') # TODO: is this always integer?
|
||||
new_options_record.scopes[type_name] = value
|
||||
offset += length
|
||||
|
||||
for field_type, length in template.option_fields.items():
|
||||
type_name = V9_FIELD_TYPES.get(field_type, None)
|
||||
is_bytes = False
|
||||
|
||||
if not type_name: # Cisco refers to the IANA IPFIX table for types >256...
|
||||
iana_type = IPFIXFieldTypes.by_id(field_type) # try to get from IPFIX types
|
||||
if iana_type:
|
||||
type_name = iana_type.name
|
||||
is_bytes = IPFIXDataTypes.is_bytes(iana_type)
|
||||
|
||||
if not type_name:
|
||||
raise ValueError
|
||||
|
||||
value = None
|
||||
if is_bytes:
|
||||
value = data[offset:offset+length]
|
||||
else:
|
||||
value = int.from_bytes(data[offset:offset+length], 'big')
|
||||
|
||||
new_options_record.data[type_name] = value
|
||||
|
||||
offset += length
|
||||
|
||||
self.option_data_records.append(new_options_record)
|
||||
|
||||
if offset % 4 == 2:
|
||||
offset += 2
|
||||
|
||||
|
||||
class V9TemplateFlowSet:
|
||||
|
|
@ -283,7 +415,7 @@ class V9TemplateFlowSet:
|
|||
|
||||
def __init__(self, data):
|
||||
pack = struct.unpack('!HH', data[:4])
|
||||
self.flowset_id = pack[0]
|
||||
self.flowset_id = pack[0] # always 0
|
||||
self.length = pack[1] # total length including this header in bytes
|
||||
self.templates = {}
|
||||
|
||||
|
|
@ -339,55 +471,90 @@ class V9Header:
|
|||
|
||||
class V9ExportPacket:
|
||||
"""The flow record holds the header and all template and data flowsets.
|
||||
|
||||
TODO: refactor into two loops: first get all contained flowsets and examine template
|
||||
flowsets first. Then data flowsets.
|
||||
"""
|
||||
|
||||
def __init__(self, data, templates):
|
||||
def __init__(self, data: bytes, templates: dict):
|
||||
self.header = V9Header(data)
|
||||
self._templates = templates
|
||||
self._new_templates = False
|
||||
self._flows = []
|
||||
self._options = []
|
||||
|
||||
offset = self.header.length
|
||||
skipped_flowsets_offsets = []
|
||||
while offset != len(data):
|
||||
flowset_id = struct.unpack('!H', data[offset:offset + 2])[0]
|
||||
|
||||
while offset != len(data):
|
||||
pack = struct.unpack('!HH', data[offset:offset + 4])
|
||||
flowset_id = pack[0] # = template id
|
||||
flowset_length = pack[1]
|
||||
|
||||
# Data template flowsets
|
||||
if flowset_id == 0: # TemplateFlowSet always have id 0
|
||||
tfs = V9TemplateFlowSet(data[offset:])
|
||||
|
||||
# Check for any new/changed templates
|
||||
if not self._new_templates:
|
||||
for id_, template in tfs.templates.items():
|
||||
if id_ not in self._templates or self._templates[id_] != template:
|
||||
self._new_templates = True
|
||||
break
|
||||
|
||||
# Update the templates with the provided templates, even if they are the same
|
||||
self._templates.update(tfs.templates)
|
||||
for id_, template in tfs.templates.items():
|
||||
if id_ not in self._templates:
|
||||
self._new_templates = True
|
||||
self._templates[id_] = template
|
||||
offset += tfs.length
|
||||
continue
|
||||
|
||||
# Option template flowsets
|
||||
elif flowset_id == 1: # Option templates always use ID 1
|
||||
# TODO: Options templates are ignored, to prevent template ID collision
|
||||
# (if a collision can occur is not yet tested)
|
||||
otfs = V9OptionsTemplateFlowSet(data[offset:])
|
||||
offset += otfs.length
|
||||
for id_, template in otfs.templates.items():
|
||||
if id_ not in self._templates:
|
||||
self._new_templates = True
|
||||
self._templates[id_] = template
|
||||
offset += otfs.flowset_length
|
||||
continue
|
||||
|
||||
# Data / option flowsets
|
||||
# First, check if template is known
|
||||
if flowset_id not in self._templates:
|
||||
# Could not be parsed, continue to check for templates
|
||||
skipped_flowsets_offsets.append(offset)
|
||||
offset += flowset_length
|
||||
continue
|
||||
|
||||
matched_template = self._templates[flowset_id]
|
||||
|
||||
if isinstance(matched_template, V9TemplateRecord):
|
||||
dfs = V9DataFlowSet(data[offset:], matched_template)
|
||||
self._flows += dfs.flows
|
||||
offset += dfs.length
|
||||
|
||||
elif isinstance(matched_template, V9OptionsTemplateRecord):
|
||||
odfs = V9OptionsDataFlowset(data[offset:], matched_template)
|
||||
self._options += odfs.option_data_records
|
||||
offset += odfs.length
|
||||
|
||||
else:
|
||||
try:
|
||||
dfs = V9DataFlowSet(data[offset:], self._templates)
|
||||
self._flows += dfs.flows
|
||||
offset += dfs.length
|
||||
except V9TemplateNotRecognized:
|
||||
# Could not be parsed, continue to check for templates
|
||||
length = struct.unpack("!H", data[offset + 2:offset + 4])[0]
|
||||
skipped_flowsets_offsets.append(offset)
|
||||
offset += length
|
||||
raise NotImplementedError
|
||||
|
||||
# In the same export packet, re-try flowsets with previously unknown templates.
|
||||
# Might happen, if an export packet first contains data flowsets, and template flowsets after
|
||||
if skipped_flowsets_offsets and self._new_templates:
|
||||
# Process flowsets in the data slice which occured before the template sets
|
||||
# Handling of offset increases is not needed here
|
||||
for offset in skipped_flowsets_offsets:
|
||||
dfs = V9DataFlowSet(data[offset:], self._templates)
|
||||
self._flows += dfs.flows
|
||||
pack = struct.unpack('!H', data[offset:offset + 2])
|
||||
flowset_id = pack[0]
|
||||
|
||||
if flowset_id not in self._templates:
|
||||
raise V9TemplateNotRecognized
|
||||
|
||||
matched_template = self._templates[flowset_id]
|
||||
if isinstance(matched_template, V9TemplateRecord):
|
||||
dfs = V9DataFlowSet(data[offset:], matched_template)
|
||||
self._flows += dfs.flows
|
||||
elif isinstance(matched_template, V9OptionsTemplateRecord):
|
||||
odfs = V9OptionsDataFlowset(data[offset:], matched_template)
|
||||
self._options += odfs.option_data_records
|
||||
|
||||
elif skipped_flowsets_offsets:
|
||||
raise V9TemplateNotRecognized
|
||||
|
||||
|
|
@ -403,7 +570,10 @@ class V9ExportPacket:
|
|||
def templates(self):
|
||||
return self._templates
|
||||
|
||||
@property
|
||||
def options(self):
|
||||
return self._options
|
||||
|
||||
def __repr__(self):
|
||||
s = " and new template(s)" if self.contains_new_templates else ""
|
||||
return "<ExportPacket v{} with {} records{}>".format(
|
||||
self.header.version, self.header.count, s)
|
||||
return "<V9ExportPacket with {} records{}>".format(self.header.count, s)
|
||||
|
|
|
|||
Loading…
Reference in a new issue