From d4d6d597133c70e1c83e3f93e100e456ee730690 Mon Sep 17 00:00:00 2001 From: Dominik Pataky Date: Mon, 30 Mar 2020 12:44:22 +0200 Subject: [PATCH] Provide parse_packet as API; fix parse_packet input handling; README To get closer to a stable package, netflow now offers the parse_packet function in its top-level __init__ file. This function was also enhanced to handle multiple input formats (str, bytes, hex bytes). Updated README accordingly. --- README.md | 42 +++++++++++++++++++++++++++++++++++------- netflow/__init__.py | 2 ++ netflow/utils.py | 20 ++++++++++++++++++-- setup.py | 2 +- 4 files changed, 56 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 9b9f29a..f1b64be 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,41 @@ # Python NetFlow library This package contains libraries and tools for **NetFlow versions 1, 5 and 9**. -Version 9 is the first NetFlow version using templates. Templates make dynamically sized and configured NetFlow data flowsets possible, which makes the collector's job harder. By importing `netflow.v1`, `netflow.v5` or `netflow.v9` you have direct access to the respective parsing objects, but at the beginning you probably will have more success by running the reference collector. +Version 9 is the first NetFlow version using templates. Templates make dynamically sized and configured NetFlow data flowsets possible, which makes the collector's job harder. The library provides the `netflow.parse_packet()` function as the main API point (see below). By importing `netflow.v1`, `netflow.v5` or `netflow.v9` you have direct access to the respective parsing objects, but at the beginning you probably will have more success by running the reference collector (example below) and look into its code. Copyright 2016-2020 Dominik Pataky Licensed under MIT License. See LICENSE. +## Using the library +If you chose to use the classes provided by this library directly, here's an example for a NetFlow v5 export packet: + + 1. Create a collector which listens for exported packets on some UDP port. It should then receive UDP packets from exporters. + 2. Inside the UDP packets, the NetFlow payload is contained. For NetFlow v5 it should begin with bytes `0005` for example. + 3. Call the `netflow.parse_packet()` function with the payload as first argument (takes string, bytes string and hex'd bytes). + +Example UDP collector server (receiving exports on port 2055): +```python +import netflow +import socket +sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +sock.bind(("0.0.0.0", 2055)) +payload, client = sock.recvfrom(4096) # experimental, tested with 1464 bytes +p = netflow.parse_packet(payload) # Test result: +print(p.header.version) # Test result: 5 +``` + +Or from hex dump: +```python +import netflow +p = netflow.parse_packet("00050003000379a35e80c58622a...") # see test_netflow.py +assert p.header.version == 5 # NetFlow v5 packet +assert p.flows[0].PROTO == 1 # ICMP flow +``` + +In NetFlow v9, templates are used instead of a fixed set of fields (like PROTO). See `collector.py` on how to handle these. + ## Using the collector and analyzer Since v0.9.0 the `netflow` library also includes reference implementations of a collector and an analyzer as CLI tools. These can be used on the CLI with `python3 -m netflow.collector` and `python3 -m netflow.analyzer`. Use the `-h` flag to receive the respective help output with all provided CLI flags. @@ -16,12 +44,12 @@ Example: to start the collector run `python3 -m netflow.collector -p 9000 -D`. T To analyze the saved traffic, run `python3 -m netflow.analyzer -f `. The output will look similar to the following snippet, with resolved hostnames and services, transferred bytes and connection duration: - 2017-10-28 23:17.01: SSH | 4.25M | 15:27 min | localmachine-2 () to localmachine-1 () - 2017-10-28 23:17.01: SSH | 4.29M | 16:22 min | remotemachine () to localmachine-2 () - 2017-10-28 23:19.01: HTTP | 22.79M | 47:32 min | uwstream3.somafm.com (173.239.76.148) to localmachine-1 () - 2017-10-28 23:22.01: HTTPS | 1.21M | 3 sec | fra16s12-in-x0e.1e100.net (2a00:1450:4001:818::200e) to localmachine-1 () - 2017-10-28 23:23.01: SSH | 93.79M | 21 sec | remotemachine () to localmachine-2 () - 2017-10-28 23:51.01: SSH | 14.08M | 1:23.09 hours | remotemachine () to localmachine-2 () + 2017-10-28 23:17.01: SSH | 4.25M | 15:27 min | local-2 () to local-1 () + 2017-10-28 23:17.01: SSH | 4.29M | 16:22 min | remote-1 () to local-2 () + 2017-10-28 23:19.01: HTTP | 22.79M | 47:32 min | uwstream3.somafm.com (173...) to local-1 () + 2017-10-28 23:22.01: HTTPS | 1.21M | 3 sec | fra16s12-in-x0e.1e100.net (2a00:..) to local-1 () + 2017-10-28 23:23.01: SSH | 93.79M | 21 sec | remote-1 () to local-2 () + 2017-10-28 23:51.01: SSH | 14.08M | 1:23.09 hours | remote-1 () to local-2 () **Please note that the collector and analyzer are experimental reference implementations. Do not rely on them in production monitoring use cases!** In any case I recommend looking into the `netflow/collector.py` and `netflow/analyzer.py` scripts for customization. Feel free to use the code and extend it in your own tool set - that's what the MIT license is for! diff --git a/netflow/__init__.py b/netflow/__init__.py index ee9f070..abb1297 100644 --- a/netflow/__init__.py +++ b/netflow/__init__.py @@ -6,3 +6,5 @@ This file belongs to https://github.com/bitkeks/python-netflow-v9-softflowd. Copyright 2017-2020 Dominik Pataky Licensed under MIT License. See LICENSE. """ + +from .utils import parse_packet diff --git a/netflow/utils.py b/netflow/utils.py index 9eb2eee..33028ac 100644 --- a/netflow/utils.py +++ b/netflow/utils.py @@ -29,7 +29,23 @@ def get_netflow_version(data): return struct.unpack('!H', data[:2])[0] -def parse_packet(data, templates): +def parse_packet(data, templates=None): + if templates is None: # compatibility for v1 and v5 + templates = {} + + if type(data) == str: + # hex dump as string + data = bytes.fromhex(data) + elif type(data) == bytes: + # check representation based on utf-8 decoding result + try: + # hex dump as bytes, but not hex + dec = data.decode() + data = bytes.fromhex(dec) + except UnicodeDecodeError: + # use data as given, assuming hex-formatted bytes + pass + version = get_netflow_version(data) if version == 1: return V1ExportPacket(data) @@ -37,4 +53,4 @@ def parse_packet(data, templates): return V5ExportPacket(data) elif version == 9: return V9ExportPacket(data, templates) - raise UnknownNetFlowVersion(data, version) \ No newline at end of file + raise UnknownNetFlowVersion(data, version) diff --git a/setup.py b/setup.py index ce2408f..0db72b3 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( url='https://github.com/bitkeks/python-netflow-v9-softflowd', packages=["netflow"], license='MIT', - python_requires='>=3', + python_requires='>=3.5', keywords='netflow collector parser', classifiers=[ "Programming Language :: Python :: 3",