diff --git a/netflow/v9.py b/netflow/v9.py index 31349c4..348deb8 100644 --- a/netflow/v9.py +++ b/netflow/v9.py @@ -15,8 +15,11 @@ Licensed under MIT License. See LICENSE. import ipaddress import struct -__all__ = ["V9DataFlowSet", "V9DataRecord", "V9ExportPacket", "V9Header", "V9TemplateField", "V9OptionsTemplateFlowSet" - "V9TemplateFlowSet", "V9TemplateNotRecognized", "V9TemplateRecord"] +from .ipfix import IPFIXFieldTypes, IPFIXDataTypes + +__all__ = ["V9DataFlowSet", "V9DataRecord", "V9ExportPacket", "V9Header", "V9TemplateField", + "V9TemplateFlowSet", "V9TemplateNotRecognized", "V9TemplateRecord", + "V9OptionsTemplateFlowSet", "V9OptionsTemplateRecord", "V9OptionsDataRecord"] V9_FIELD_TYPES = { 0: 'UNKNOWN_FIELD_TYPE', # fallback for unknown field types @@ -155,6 +158,14 @@ V9_FIELD_TYPES = { 56702: 'PANOS_USERID' } +V9_SCOPE_TYPES = { + 1: "System", + 2: "Interface", + 3: "Line Card", + 4: "Cache", + 5: "Template" +} + class V9TemplateNotRecognized(KeyError): pass @@ -181,7 +192,7 @@ class V9DataFlowSet: DataFlowSet and must not be zero. """ - def __init__(self, data, templates): + def __init__(self, data, template): pack = struct.unpack('!HH', data[:4]) self.template_id = pack[0] # flowset_id is reference to a template_id @@ -190,11 +201,6 @@ class V9DataFlowSet: offset = 4 - if self.template_id not in templates: - raise V9TemplateNotRecognized - - template = templates[self.template_id] - # As the field lengths are variable V9 has padding to next 32 Bit padding_size = 4 - (self.length % 4) # 4 Byte @@ -252,7 +258,7 @@ class V9TemplateRecord: """A template record contained in a TemplateFlowSet. """ - def __init__(self, template_id, field_count, fields): + def __init__(self, template_id, field_count, fields: list): self.template_id = template_id self.field_count = field_count self.fields = fields @@ -263,15 +269,141 @@ class V9TemplateRecord: ' '.join([V9_FIELD_TYPES[field.field_type] for field in self.fields])) -class V9OptionsTemplateFlowSet: - """An options template flowset. Always uses flowset ID 1. - TODO: not handled at the moment, only stub implementation +class V9OptionsDataRecord: + def __init__(self): + self.scopes = {} + self.data = {} + + def __repr__(self): + return "".format(self.scopes.keys(), self.data.keys()) + + +class V9OptionsTemplateRecord: + """An options template record contained in an options template flowset. """ - def __init__(self, data): - pack = struct.unpack('!HHH', data[:6]) - self.flowset_id = pack[0] + + def __init__(self, template_id, scope_fields: dict, option_fields: dict): + self.template_id = template_id + self.scope_fields = scope_fields + self.option_fields = option_fields + + def __repr__(self): + return "".format( + self.scope_fields.keys(), self.option_fields.keys()) + + +class V9OptionsTemplateFlowSet: + """An options template flowset. + + > Each Options Template FlowSet MAY contain multiple Options Template Records. + + Scope field types range from 1 to 5: + 1 System + 2 Interface + 3 Line Card + 4 Cache + 5 Template + """ + + def __init__(self, data: bytes): + pack = struct.unpack('!HH', data[:4]) + self.flowset_id = pack[0] # always 1 + self.flowset_length = pack[1] # length of this flowset + self.templates = {} + + offset = 4 + + while offset < self.flowset_length: + pack = struct.unpack("!HHH", data[offset:offset + 6]) # options template header + template_id = pack[0] # value above 255 + option_scope_length = pack[1] + options_length = pack[2] + + offset += 6 + + # Fetch all scope fields (most probably only one field) + scopes = {} # Holds "type: length" key-value pairs + + if option_scope_length % 4 != 0 or options_length % 4 != 0: + raise ValueError(option_scope_length, options_length) + + for scope_counter in range(option_scope_length // 4): # example: option_scope_length = 4 means one scope + pack = struct.unpack("!HH", data[offset:offset + 4]) + scope_field_type = pack[0] # values range from 1 to 5 + scope_field_length = pack[1] + scopes[scope_field_type] = scope_field_length + offset += 4 + + # Fetch all option fields + options = {} # same + for option_counter in range(options_length // 4): # now counting the options + pack = struct.unpack("!HH", data[offset:offset + 4]) + option_field_type = pack[0] + option_field_length = pack[1] + options[option_field_type] = option_field_length + offset += 4 + + optionstemplate = V9OptionsTemplateRecord(template_id, scopes, options) + + self.templates[template_id] = optionstemplate + + # handle padding and add offset if needed + if offset % 4 == 2: + offset += 2 + + def __repr__(self): + return "".format(len(self.templates), self.templates.keys()) + + +class V9OptionsDataFlowset: + """An options data flowset with option data records + """ + + def __init__(self, data: bytes, template: V9OptionsTemplateRecord): + pack = struct.unpack('!HH', data[:4]) + + self.template_id = pack[0] self.length = pack[1] - self.template_id = pack[2] + self.option_data_records = [] + + offset = 4 + + while offset < self.length: + new_options_record = V9OptionsDataRecord() + + for scope_type, length in template.scope_fields.items(): + type_name = V9_SCOPE_TYPES.get(scope_type, scope_type) # Either name, or unknown int + value = int.from_bytes(data[offset:offset+length], 'big') # TODO: is this always integer? + new_options_record.scopes[type_name] = value + offset += length + + for field_type, length in template.option_fields.items(): + type_name = V9_FIELD_TYPES.get(field_type, None) + is_bytes = False + + if not type_name: # Cisco refers to the IANA IPFIX table for types >256... + iana_type = IPFIXFieldTypes.by_id(field_type) # try to get from IPFIX types + if iana_type: + type_name = iana_type.name + is_bytes = IPFIXDataTypes.is_bytes(iana_type) + + if not type_name: + raise ValueError + + value = None + if is_bytes: + value = data[offset:offset+length] + else: + value = int.from_bytes(data[offset:offset+length], 'big') + + new_options_record.data[type_name] = value + + offset += length + + self.option_data_records.append(new_options_record) + + if offset % 4 == 2: + offset += 2 class V9TemplateFlowSet: @@ -283,7 +415,7 @@ class V9TemplateFlowSet: def __init__(self, data): pack = struct.unpack('!HH', data[:4]) - self.flowset_id = pack[0] + self.flowset_id = pack[0] # always 0 self.length = pack[1] # total length including this header in bytes self.templates = {} @@ -339,55 +471,90 @@ class V9Header: class V9ExportPacket: """The flow record holds the header and all template and data flowsets. + + TODO: refactor into two loops: first get all contained flowsets and examine template + flowsets first. Then data flowsets. """ - def __init__(self, data, templates): + def __init__(self, data: bytes, templates: dict): self.header = V9Header(data) self._templates = templates self._new_templates = False self._flows = [] + self._options = [] offset = self.header.length skipped_flowsets_offsets = [] - while offset != len(data): - flowset_id = struct.unpack('!H', data[offset:offset + 2])[0] + while offset != len(data): + pack = struct.unpack('!HH', data[offset:offset + 4]) + flowset_id = pack[0] # = template id + flowset_length = pack[1] + + # Data template flowsets if flowset_id == 0: # TemplateFlowSet always have id 0 tfs = V9TemplateFlowSet(data[offset:]) - - # Check for any new/changed templates - if not self._new_templates: - for id_, template in tfs.templates.items(): - if id_ not in self._templates or self._templates[id_] != template: - self._new_templates = True - break - # Update the templates with the provided templates, even if they are the same - self._templates.update(tfs.templates) + for id_, template in tfs.templates.items(): + if id_ not in self._templates: + self._new_templates = True + self._templates[id_] = template offset += tfs.length + continue + # Option template flowsets elif flowset_id == 1: # Option templates always use ID 1 - # TODO: Options templates are ignored, to prevent template ID collision - # (if a collision can occur is not yet tested) otfs = V9OptionsTemplateFlowSet(data[offset:]) - offset += otfs.length + for id_, template in otfs.templates.items(): + if id_ not in self._templates: + self._new_templates = True + self._templates[id_] = template + offset += otfs.flowset_length + continue + + # Data / option flowsets + # First, check if template is known + if flowset_id not in self._templates: + # Could not be parsed, continue to check for templates + skipped_flowsets_offsets.append(offset) + offset += flowset_length + continue + + matched_template = self._templates[flowset_id] + + if isinstance(matched_template, V9TemplateRecord): + dfs = V9DataFlowSet(data[offset:], matched_template) + self._flows += dfs.flows + offset += dfs.length + + elif isinstance(matched_template, V9OptionsTemplateRecord): + odfs = V9OptionsDataFlowset(data[offset:], matched_template) + self._options += odfs.option_data_records + offset += odfs.length else: - try: - dfs = V9DataFlowSet(data[offset:], self._templates) - self._flows += dfs.flows - offset += dfs.length - except V9TemplateNotRecognized: - # Could not be parsed, continue to check for templates - length = struct.unpack("!H", data[offset + 2:offset + 4])[0] - skipped_flowsets_offsets.append(offset) - offset += length + raise NotImplementedError + # In the same export packet, re-try flowsets with previously unknown templates. + # Might happen, if an export packet first contains data flowsets, and template flowsets after if skipped_flowsets_offsets and self._new_templates: # Process flowsets in the data slice which occured before the template sets + # Handling of offset increases is not needed here for offset in skipped_flowsets_offsets: - dfs = V9DataFlowSet(data[offset:], self._templates) - self._flows += dfs.flows + pack = struct.unpack('!H', data[offset:offset + 2]) + flowset_id = pack[0] + + if flowset_id not in self._templates: + raise V9TemplateNotRecognized + + matched_template = self._templates[flowset_id] + if isinstance(matched_template, V9TemplateRecord): + dfs = V9DataFlowSet(data[offset:], matched_template) + self._flows += dfs.flows + elif isinstance(matched_template, V9OptionsTemplateRecord): + odfs = V9OptionsDataFlowset(data[offset:], matched_template) + self._options += odfs.option_data_records + elif skipped_flowsets_offsets: raise V9TemplateNotRecognized @@ -403,7 +570,10 @@ class V9ExportPacket: def templates(self): return self._templates + @property + def options(self): + return self._options + def __repr__(self): s = " and new template(s)" if self.contains_new_templates else "" - return "".format( - self.header.version, self.header.count, s) + return "".format(self.header.count, s)