dprevot/netflow
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Extend and reformat tests, add tests for v1 and v5, bump version

Browse source 
The tests are now also parsing export packets for version 1 and 5.
Version 9 received an additional test, inspecting the data inside the
export.

All new packet hex dumps were created by using a Docker container with
alpine Linux, running a softflowd daemon inside and then pinging the
Docker host IP. After review with "softflowctl dump-flows" issueing
"softflowctl expire-all" sends the packets away to the collector (should
be an IP address outside of the Docker bridge). The export network
packets are then collected with Wireshark running in the host namespace,
capturing on the Docker bridge.

Bump version to v0.8.3

Resolves #13
Resolves #14
Refs #18
This commit is contained in:
Dominik Pataky 2020-03-29 19:52:08 +02:00
parent e8073013c1
commit 9d2bc21ae2
2 changed files with 147 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
setup.py
View file

@ -7,7 +7,7 @@ with open("README.md", "r") as fh:
setup(
name='netflow',
version='0.8.2',
version='0.8.3',
description='NetFlow v1, v5, and v9 tool suite implemented in Python 3.',
long_description=long_description,
long_description_content_type='text/markdown',

167
tests.py
View file

@ -1,15 +1,15 @@
#!/usr/bin/env python3
"""
This file contains tests for the softflowd UDP collector saved in main.py The
test packets (defined below as hex streams) were extracted from a "real"
softflowd export based on a sample PCAP capture file. They consist of one
export with the templates and three without.
This file contains tests for the NetFlow collector for versions 1, 5 and 9.
The test packets (defined below as hex streams) were extracted from "real"
softflowd exports based on a sample PCAP capture file.
Copyright 2017-2020 Dominik Pataky <dev@bitkeks.eu>
Licensed under MIT License. See LICENSE.
"""
import gzip
import ipaddress
import json
import queue
import random
@ -21,23 +21,84 @@ import unittest
from main import NetFlowListener
# TODO: add tests for v1 and v5
# TODO: tests with 500 packets fail?
# TODO: tests with 500 packets fail? Probably a problem with UDP sockets
# The flowset with 2 templates and 8 flows
TEMPLATE_PACKET = '0009000a000000035c9f55980000000100000000000000400400000e00080004000c000400150004001600040001000400020004000a0004000e000400070002000b00020004000100060001003c000100050001000000400800000e001b0010001c001000150004001600040001000400020004000a0004000e000400070002000b00020004000100060001003c000100050001040001447f0000017f000001fb3c1aaafb3c18fd000190100000004b00000000000000000050942c061b04007f0000017f000001fb3c1aaafb3c18fd00000f94000000360000000000000000942c0050061f04007f0000017f000001fb3c1cfcfb3c1a9b0000d3fc0000002a000000000000000000509434061b04007f0000017f000001fb3c1cfcfb3c1a9b00000a490000001e000000000000000094340050061f04007f0000017f000001fb3bb82cfb3ba48b000002960000000300000000000000000050942a061904007f0000017f000001fb3bb82cfb3ba48b00000068000000020000000000000000942a0050061104007f0000017f000001fb3c1900fb3c18fe0000004c0000000100000000000000000035b3c9110004007f0000017f000001fb3c1900fb3c18fe0000003c000000010000000000000000b3c9003511000400'
# The flowset with 2 templates (IPv4 and IPv6) and 8 flows with data
PACKET_V9_TEMPLATE = "0009000a000000035c9f55980000000100000000000000400400000e00080004000c000400150004" \
"001600040001000400020004000a0004000e000400070002000b00020004000100060001003c0001" \
"00050001000000400800000e001b0010001c001000150004001600040001000400020004000a0004" \
"000e000400070002000b00020004000100060001003c000100050001040001447f0000017f000001" \
"fb3c1aaafb3c18fd000190100000004b00000000000000000050942c061b04007f0000017f000001" \
"fb3c1aaafb3c18fd00000f94000000360000000000000000942c0050061f04007f0000017f000001" \
"fb3c1cfcfb3c1a9b0000d3fc0000002a000000000000000000509434061b04007f0000017f000001" \
"fb3c1cfcfb3c1a9b00000a490000001e000000000000000094340050061f04007f0000017f000001" \
"fb3bb82cfb3ba48b000002960000000300000000000000000050942a061904007f0000017f000001" \
"fb3bb82cfb3ba48b00000068000000020000000000000000942a0050061104007f0000017f000001" \
"fb3c1900fb3c18fe0000004c0000000100000000000000000035b3c9110004007f0000017f000001" \
"fb3c1900fb3c18fe0000003c000000010000000000000000b3c9003511000400"
# Three packets without templates, each with 12 flows, anonymized
PACKETS = [
'0009000c000000035c9f55980000000200000000040001e47f0000017f000001fb3c1a17fb3c19fd000001480000000200000000000000000035ea82110004007f0000017f000001fb3c1a17fb3c19fd0000007a000000020000000000000000ea820035110004007f0000017f000001fb3c1a17fb3c19fd000000f80000000200000000000000000035c6e2110004007f0000017f000001fb3c1a17fb3c19fd0000007a000000020000000000000000c6e20035110004007f0000017f000001fb3c1a9efb3c1a9c0000004c0000000100000000000000000035adc1110004007f0000017f000001fb3c1a9efb3c1a9c0000003c000000010000000000000000adc10035110004007f0000017f000001fb3c1b74fb3c1b720000004c0000000100000000000000000035d0b3110004007f0000017f000001fb3c1b74fb3c1b720000003c000000010000000000000000d0b30035110004007f0000017f000001fb3c2f59fb3c1b7100001a350000000a000000000000000000509436061b04007f0000017f000001fb3c2f59fb3c1b710000038a0000000a000000000000000094360050061b04007f0000017f000001fb3c913bfb3c91380000004c0000000100000000000000000035e262110004007f0000017f000001fb3c913bfb3c91380000003c000000010000000000000000e262003511000400',
'0009000c000000035c9f55980000000300000000040001e47f0000017f000001fb3ca523fb3c913b0000030700000005000000000000000000509438061b04007f0000017f000001fb3ca523fb3c913b000002a200000005000000000000000094380050061b04007f0000017f000001fb3f7fe1fb3dbc970002d52800000097000000000000000001bb8730061b04007f0000017f000001fb3f7fe1fb3dbc970000146c000000520000000000000000873001bb061f04007f0000017f000001fb3d066ffb3d066c0000004c0000000100000000000000000035e5bd110004007f0000017f000001fb3d066ffb3d066c0000003c000000010000000000000000e5bd0035110004007f0000017f000001fb3d1a61fb3d066b000003060000000500000000000000000050943a061b04007f0000017f000001fb3d1a61fb3d066b000002a2000000050000000000000000943a0050061b04007f0000017f000001fb3fed00fb3f002c0000344000000016000000000000000001bbae50061f04007f0000017f000001fb3fed00fb3f002c00000a47000000120000000000000000ae5001bb061b04007f0000017f000001fb402f17fb402a750003524c000000a5000000000000000001bbc48c061b04007f0000017f000001fb402f17fb402a75000020a60000007e0000000000000000c48c01bb061f0400',
'0009000c000000035c9f55980000000400000000040001e47f0000017f000001fb3d7ba2fb3d7ba00000004c0000000100000000000000000035a399110004007f0000017f000001fb3d7ba2fb3d7ba00000003c000000010000000000000000a3990035110004007f0000017f000001fb3d8f85fb3d7b9f000003070000000500000000000000000050943c061b04007f0000017f000001fb3d8f85fb3d7b9f000002a2000000050000000000000000943c0050061b04007f0000017f000001fb3d9165fb3d7f6d0000c97b0000002a000000000000000001bbae48061b04007f0000017f000001fb3d9165fb3d7f6d000007f40000001a0000000000000000ae4801bb061b04007f0000017f000001fb3dbc96fb3dbc7e0000011e0000000200000000000000000035bd4f110004007f0000017f000001fb3dbc96fb3dbc7e0000008e000000020000000000000000bd4f0035110004007f0000017f000001fb3ddbb3fb3c1a180000bfee0000002f00000000000000000050ae56061b04007f0000017f000001fb3ddbb3fb3c1a1800000982000000270000000000000000ae560050061b04007f0000017f000001fb3ddbb3fb3c1a180000130e0000001200000000000000000050e820061b04007f0000017f000001fb3ddbb3fb3c1a180000059c000000140000000000000000e8200050061b0400'
PACKETS_V9 = [
"0009000c000000035c9f55980000000200000000040001e47f0000017f000001fb3c1a17fb3c19fd"
"000001480000000200000000000000000035ea82110004007f0000017f000001fb3c1a17fb3c19fd"
"0000007a000000020000000000000000ea820035110004007f0000017f000001fb3c1a17fb3c19fd"
"000000f80000000200000000000000000035c6e2110004007f0000017f000001fb3c1a17fb3c19fd"
"0000007a000000020000000000000000c6e20035110004007f0000017f000001fb3c1a9efb3c1a9c"
"0000004c0000000100000000000000000035adc1110004007f0000017f000001fb3c1a9efb3c1a9c"
"0000003c000000010000000000000000adc10035110004007f0000017f000001fb3c1b74fb3c1b72"
"0000004c0000000100000000000000000035d0b3110004007f0000017f000001fb3c1b74fb3c1b72"
"0000003c000000010000000000000000d0b30035110004007f0000017f000001fb3c2f59fb3c1b71"
"00001a350000000a000000000000000000509436061b04007f0000017f000001fb3c2f59fb3c1b71"
"0000038a0000000a000000000000000094360050061b04007f0000017f000001fb3c913bfb3c9138"
"0000004c0000000100000000000000000035e262110004007f0000017f000001fb3c913bfb3c9138"
"0000003c000000010000000000000000e262003511000400",
"0009000c000000035c9f55980000000300000000040001e47f0000017f000001fb3ca523fb3c913b"
"0000030700000005000000000000000000509438061b04007f0000017f000001fb3ca523fb3c913b"
"000002a200000005000000000000000094380050061b04007f0000017f000001fb3f7fe1fb3dbc97"
"0002d52800000097000000000000000001bb8730061b04007f0000017f000001fb3f7fe1fb3dbc97"
"0000146c000000520000000000000000873001bb061f04007f0000017f000001fb3d066ffb3d066c"
"0000004c0000000100000000000000000035e5bd110004007f0000017f000001fb3d066ffb3d066c"
"0000003c000000010000000000000000e5bd0035110004007f0000017f000001fb3d1a61fb3d066b"
"000003060000000500000000000000000050943a061b04007f0000017f000001fb3d1a61fb3d066b"
"000002a2000000050000000000000000943a0050061b04007f0000017f000001fb3fed00fb3f002c"
"0000344000000016000000000000000001bbae50061f04007f0000017f000001fb3fed00fb3f002c"
"00000a47000000120000000000000000ae5001bb061b04007f0000017f000001fb402f17fb402a75"
"0003524c000000a5000000000000000001bbc48c061b04007f0000017f000001fb402f17fb402a75"
"000020a60000007e0000000000000000c48c01bb061f0400",
"0009000c000000035c9f55980000000400000000040001e47f0000017f000001fb3d7ba2fb3d7ba0"
"0000004c0000000100000000000000000035a399110004007f0000017f000001fb3d7ba2fb3d7ba0"
"0000003c000000010000000000000000a3990035110004007f0000017f000001fb3d8f85fb3d7b9f"
"000003070000000500000000000000000050943c061b04007f0000017f000001fb3d8f85fb3d7b9f"
"000002a2000000050000000000000000943c0050061b04007f0000017f000001fb3d9165fb3d7f6d"
"0000c97b0000002a000000000000000001bbae48061b04007f0000017f000001fb3d9165fb3d7f6d"
"000007f40000001a0000000000000000ae4801bb061b04007f0000017f000001fb3dbc96fb3dbc7e"
"0000011e0000000200000000000000000035bd4f110004007f0000017f000001fb3dbc96fb3dbc7e"
"0000008e000000020000000000000000bd4f0035110004007f0000017f000001fb3ddbb3fb3c1a18"
"0000bfee0000002f00000000000000000050ae56061b04007f0000017f000001fb3ddbb3fb3c1a18"
"00000982000000270000000000000000ae560050061b04007f0000017f000001fb3ddbb3fb3c1a18"
"0000130e0000001200000000000000000050e820061b04007f0000017f000001fb3ddbb3fb3c1a18"
"0000059c000000140000000000000000e8200050061b0400"
]
INVALID_PACKET = "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"
# Example export for v1 which contains two flows from one ICMP ping request/reply session
PACKET_V1 = "000100020001189b5e80c32c2fd41848ac110002ac11000100000000000000000000000a00000348" \
"000027c700004af100000800000001000000000000000000ac110001ac1100020000000000000000" \
"0000000a00000348000027c700004af100000000000001000000000000000000"
# Example export for v5 which contains three flows, two for ICMP ping and one multicast on interface (224.0.0.251)
PACKET_V5 = "00050003000379a35e80c58622a55ab00000000000000000ac110002ac1100010000000000000000" \
"0000000a0000034800002f4c0000527600000800000001000000000000000000ac110001ac110002" \
"00000000000000000000000a0000034800002f4c0000527600000000000001000000000000000000" \
"ac110001e00000fb000000000000000000000001000000a90000e01c0000e01c14e914e900001100" \
"0000000000000000"
# Invalid export hex stream
PACKET_INVALID = "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"
CONNECTION = ('127.0.0.1', 1337)
NUM_PACKETS = 50
NUM_PACKETS = 100
def emit_packets(packets, delay=0):
@ -49,7 +110,7 @@ def emit_packets(packets, delay=0):
sock.close()
def send_recv_packets(packets, delay=0):
def send_recv_packets(packets, delay=0) -> (list, float, float):
"""Starts a listener, send packets, receives packets
returns a tuple: ([(ts, export), ...], time_started_sending, time_stopped_sending)
@ -72,16 +133,16 @@ def send_recv_packets(packets, delay=0):
return pkts, tstart, tend
class TestSoftFlowExport(unittest.TestCase):
class TestFlowExport(unittest.TestCase):
def _test_recv_all_packets(self, num, template_idx, delay=0):
"""Fling packets at the server and test that it receives them all"""
def gen_pkts(n, idx):
for x in range(n):
if x == idx:
yield TEMPLATE_PACKET
yield PACKET_V9_TEMPLATE
else:
yield random.choice(PACKETS)
yield random.choice(PACKETS_V9)
pkts, tstart, tend = send_recv_packets(gen_pkts(num, template_idx), delay=delay)
@ -121,15 +182,79 @@ class TestSoftFlowExport(unittest.TestCase):
"""Test that invalid packets log a warning but are otherwise ignored"""
with self.assertLogs(level='WARNING'):
pkts, _, _ = send_recv_packets([
INVALID_PACKET, TEMPLATE_PACKET, random.choice(PACKETS), INVALID_PACKET,
random.choice(PACKETS), INVALID_PACKET
PACKET_INVALID, PACKET_V9_TEMPLATE, random.choice(PACKETS_V9), PACKET_INVALID,
random.choice(PACKETS_V9), PACKET_INVALID
])
self.assertEqual(len(pkts), 3)
def test_recv_v1_packet(self):
"""Test NetFlow v1 packet parsing"""
pkts, _, _ = send_recv_packets([PACKET_V1])
self.assertEqual(len(pkts), 1)
# Take the parsed packet and check meta data
p = pkts[0]
self.assertEqual(p.client[0], "127.0.0.1") # collector listens locally
self.assertEqual(len(p.export.flows), 2) # ping request and reply
self.assertEqual(p.export.header.count, 2) # same value, in header
self.assertEqual(p.export.header.version, 1)
# Check specific IP address contained in a flow.
# Since it might vary which flow of the pair is epxorted first, check both
flow = p.export.flows[0]
self.assertIn(
ipaddress.ip_address(flow.data["IPV4_SRC_ADDR"]), # convert to ipaddress obj because value is int
[ipaddress.ip_address("172.17.0.1"), ipaddress.ip_address("172.17.0.2")]
)
def test_recv_v5_packet(self):
"""Test NetFlow v5 packet parsing"""
pkts, _, _ = send_recv_packets([PACKET_V5])
self.assertEqual(len(pkts), 1)
p = pkts[0]
self.assertEqual(p.client[0], "127.0.0.1")
self.assertEqual(len(p.export.flows), 3) # ping request and reply, one multicast
self.assertEqual(p.export.header.count, 3)
self.assertEqual(p.export.header.version, 5)
# Check specific IP address contained in a flow.
# Since it might vary which flow of the pair is epxorted first, check both
flow = p.export.flows[0]
self.assertIn(
ipaddress.ip_address(flow.data["IPV4_SRC_ADDR"]), # convert to ipaddress obj because value is int
[ipaddress.ip_address("172.17.0.1"), ipaddress.ip_address("172.17.0.2")] # matches multicast packet too
)
def test_recv_v9_packet(self):
"""Test NetFlow v9 packet parsing"""
# send packet without any template, must fail to parse (packets are queued)
pkts, _, _ = send_recv_packets([PACKETS_V9[0]])
self.assertEqual(len(pkts), 0) # no export is parsed due to missing template
# send packet with two templates and eight flows, should succeed
pkts, _, _ = send_recv_packets([PACKET_V9_TEMPLATE])
self.assertEqual(len(pkts), 1)
p = pkts[0]
self.assertEqual(p.client[0], "127.0.0.1")
self.assertEqual(len(p.export.flows), 8) # count flows
self.assertEqual(len(p.export.templates), 2) # count new templates
# send template and multiple export packets
pkts, _, _ = send_recv_packets([PACKET_V9_TEMPLATE, *PACKETS_V9])
self.assertEqual(len(pkts), 4)
self.assertEqual(pkts[0].export.header.version, 9)
total_flows = 0
for packet in pkts:
total_flows += len(packet.export.flows)
self.assertEqual(total_flows, 8 + 12 + 12 + 12)
@unittest.skip("Test is not adapted to current analyzer script")
def test_analyzer(self):
"""Test that the analyzer doesn't break and outputs the correct number of lines"""
pkts, _, _ = send_recv_packets([TEMPLATE_PACKET, *PACKETS])
pkts, _, _ = send_recv_packets([PACKET_V9_TEMPLATE, *PACKETS_V9])
data = {p.ts: [f.data for f in p.export.flows] for p in pkts}
# Different stdout/stderr arguments for backwards compatibility