From 547792c5c2cecfccdd8a2b29011461dd2b69db07 Mon Sep 17 00:00:00 2001 From: Dominik Pataky Date: Wed, 1 Apr 2020 14:15:53 +0200 Subject: [PATCH] Tests: move packets into each version test file; add tests for IPFIX The previously introduced tests/lib.py contained the NetFlow v9 packets and then the IPFIX packets, those were split and put into their respective test files again. The lib now contains shared objects only. For IPFIX tests were added. Two new packets were added, one with templates and one without (again, real exports from softflowd). Different cases are checked: no template, template and later template. Fields of flows are also checked, especially IPv6 addresses. Note: exports made with softflowd were created by softflowd 1.0.0, compiled from https://github.com/irino/softflowd --- tests/__init__.py | 0 tests/lib.py | 86 ---------------------------------------- tests/test_ipfix.py | 91 ++++++++++++++++++++++++++++++++++++++++++- tests/test_netflow.py | 88 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 88 deletions(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/lib.py b/tests/lib.py index f769460..041831a 100644 --- a/tests/lib.py +++ b/tests/lib.py @@ -15,92 +15,6 @@ import time from netflow.collector import ThreadedNetFlowListener -PACKET_V9_TEMPLATE = "0009000a000000035c9f55980000000100000000000000400400000e00080004000c000400150004" \ - "001600040001000400020004000a0004000e000400070002000b00020004000100060001003c0001" \ - "00050001000000400800000e001b0010001c001000150004001600040001000400020004000a0004" \ - "000e000400070002000b00020004000100060001003c000100050001040001447f0000017f000001" \ - "fb3c1aaafb3c18fd000190100000004b00000000000000000050942c061b04007f0000017f000001" \ - "fb3c1aaafb3c18fd00000f94000000360000000000000000942c0050061f04007f0000017f000001" \ - "fb3c1cfcfb3c1a9b0000d3fc0000002a000000000000000000509434061b04007f0000017f000001" \ - "fb3c1cfcfb3c1a9b00000a490000001e000000000000000094340050061f04007f0000017f000001" \ - "fb3bb82cfb3ba48b000002960000000300000000000000000050942a061904007f0000017f000001" \ - "fb3bb82cfb3ba48b00000068000000020000000000000000942a0050061104007f0000017f000001" \ - "fb3c1900fb3c18fe0000004c0000000100000000000000000035b3c9110004007f0000017f000001" \ - "fb3c1900fb3c18fe0000003c000000010000000000000000b3c9003511000400" - -# This packet is special. We take PACKET_V9_TEMPLATE and re-order the templates and flows. -# The first line is the header, the smaller lines the templates and the long lines the flows (limited to 80 chars) -PACKET_V9_TEMPLATE_MIXED = ("0009000a000000035c9f55980000000100000000" # header - "040001447f0000017f000001fb3c1aaafb3c18fd000190100000004b00000000000000000050942c" - "061b04007f0000017f000001fb3c1aaafb3c18fd00000f94000000360000000000000000942c0050" - "061f04007f0000017f000001fb3c1cfcfb3c1a9b0000d3fc0000002a000000000000000000509434" - "061b04007f0000017f000001fb3c1cfcfb3c1a9b00000a490000001e000000000000000094340050" - "061f04007f0000017f000001fb3bb82cfb3ba48b000002960000000300000000000000000050942a" - "061904007f0000017f000001fb3bb82cfb3ba48b00000068000000020000000000000000942a0050" - "061104007f0000017f000001fb3c1900fb3c18fe0000004c0000000100000000000000000035b3c9" - "110004007f0000017f000001fb3c1900fb3c18fe0000003c000000010000000000000000b3c90035" - "11000400" # end of flow segments - "000000400400000e00080004000c000400150004001600040001000400020004" # template 1024 - "000a0004000e000400070002000b00020004000100060001003c000100050001" - "000000400800000e001b0010001c001000150004001600040001000400020004" # template 2048 - "000a0004000e000400070002000b00020004000100060001003c000100050001") - -# Three packets without templates, each with 12 flows, anonymized -PACKETS_V9 = [ - "0009000c000000035c9f55980000000200000000040001e47f0000017f000001fb3c1a17fb3c19fd" - "000001480000000200000000000000000035ea82110004007f0000017f000001fb3c1a17fb3c19fd" - "0000007a000000020000000000000000ea820035110004007f0000017f000001fb3c1a17fb3c19fd" - "000000f80000000200000000000000000035c6e2110004007f0000017f000001fb3c1a17fb3c19fd" - "0000007a000000020000000000000000c6e20035110004007f0000017f000001fb3c1a9efb3c1a9c" - "0000004c0000000100000000000000000035adc1110004007f0000017f000001fb3c1a9efb3c1a9c" - "0000003c000000010000000000000000adc10035110004007f0000017f000001fb3c1b74fb3c1b72" - "0000004c0000000100000000000000000035d0b3110004007f0000017f000001fb3c1b74fb3c1b72" - "0000003c000000010000000000000000d0b30035110004007f0000017f000001fb3c2f59fb3c1b71" - "00001a350000000a000000000000000000509436061b04007f0000017f000001fb3c2f59fb3c1b71" - "0000038a0000000a000000000000000094360050061b04007f0000017f000001fb3c913bfb3c9138" - "0000004c0000000100000000000000000035e262110004007f0000017f000001fb3c913bfb3c9138" - "0000003c000000010000000000000000e262003511000400", - - "0009000c000000035c9f55980000000300000000040001e47f0000017f000001fb3ca523fb3c913b" - "0000030700000005000000000000000000509438061b04007f0000017f000001fb3ca523fb3c913b" - "000002a200000005000000000000000094380050061b04007f0000017f000001fb3f7fe1fb3dbc97" - "0002d52800000097000000000000000001bb8730061b04007f0000017f000001fb3f7fe1fb3dbc97" - "0000146c000000520000000000000000873001bb061f04007f0000017f000001fb3d066ffb3d066c" - "0000004c0000000100000000000000000035e5bd110004007f0000017f000001fb3d066ffb3d066c" - "0000003c000000010000000000000000e5bd0035110004007f0000017f000001fb3d1a61fb3d066b" - "000003060000000500000000000000000050943a061b04007f0000017f000001fb3d1a61fb3d066b" - "000002a2000000050000000000000000943a0050061b04007f0000017f000001fb3fed00fb3f002c" - "0000344000000016000000000000000001bbae50061f04007f0000017f000001fb3fed00fb3f002c" - "00000a47000000120000000000000000ae5001bb061b04007f0000017f000001fb402f17fb402a75" - "0003524c000000a5000000000000000001bbc48c061b04007f0000017f000001fb402f17fb402a75" - "000020a60000007e0000000000000000c48c01bb061f0400", - - "0009000c000000035c9f55980000000400000000040001e47f0000017f000001fb3d7ba2fb3d7ba0" - "0000004c0000000100000000000000000035a399110004007f0000017f000001fb3d7ba2fb3d7ba0" - "0000003c000000010000000000000000a3990035110004007f0000017f000001fb3d8f85fb3d7b9f" - "000003070000000500000000000000000050943c061b04007f0000017f000001fb3d8f85fb3d7b9f" - "000002a2000000050000000000000000943c0050061b04007f0000017f000001fb3d9165fb3d7f6d" - "0000c97b0000002a000000000000000001bbae48061b04007f0000017f000001fb3d9165fb3d7f6d" - "000007f40000001a0000000000000000ae4801bb061b04007f0000017f000001fb3dbc96fb3dbc7e" - "0000011e0000000200000000000000000035bd4f110004007f0000017f000001fb3dbc96fb3dbc7e" - "0000008e000000020000000000000000bd4f0035110004007f0000017f000001fb3ddbb3fb3c1a18" - "0000bfee0000002f00000000000000000050ae56061b04007f0000017f000001fb3ddbb3fb3c1a18" - "00000982000000270000000000000000ae560050061b04007f0000017f000001fb3ddbb3fb3c1a18" - "0000130e0000001200000000000000000050e820061b04007f0000017f000001fb3ddbb3fb3c1a18" - "0000059c000000140000000000000000e8200050061b0400" -] - -# Example export for v1 which contains two flows from one ICMP ping request/reply session -PACKET_V1 = "000100020001189b5e80c32c2fd41848ac110002ac11000100000000000000000000000a00000348" \ - "000027c700004af100000800000001000000000000000000ac110001ac1100020000000000000000" \ - "0000000a00000348000027c700004af100000000000001000000000000000000" - -# Example export for v5 which contains three flows, two for ICMP ping and one multicast on interface (224.0.0.251) -PACKET_V5 = "00050003000379a35e80c58622a55ab00000000000000000ac110002ac1100010000000000000000" \ - "0000000a0000034800002f4c0000527600000800000001000000000000000000ac110001ac110002" \ - "00000000000000000000000a0000034800002f4c0000527600000000000001000000000000000000" \ - "ac110001e00000fb000000000000000000000001000000a90000e01c0000e01c14e914e900001100" \ - "0000000000000000" # Invalid export hex stream PACKET_INVALID = "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" diff --git a/tests/test_ipfix.py b/tests/test_ipfix.py index 06afa7e..db4ab3d 100644 --- a/tests/test_ipfix.py +++ b/tests/test_ipfix.py @@ -9,11 +9,98 @@ Licensed under MIT License. See LICENSE. # TODO: tests with 500 packets fail with delay=0. Probably a problem with UDP sockets buffer import ipaddress -import random import unittest from tests.lib import * +# Example export for IPFIX (v10) with 4 templates, 1 option template and 8 data flow sets +PACKET_IPFIX_TEMPLATE = "000a05205e8465fd0000001300000000000200400400000e00080004000c00040016000400150004" \ + "0001000400020004000a0004000e000400070002000b00020004000100060001003c000100050001" \ + "000200340401000b00080004000c000400160004001500040001000400020004000a0004000e0004" \ + "00200002003c000100050001000200400800000e001b0010001c0010001600040015000400010004" \ + "00020004000a0004000e000400070002000b00020004000100060001003c00010005000100020034" \ + "0801000b001b0010001c001000160004001500040001000400020004000a0004000e0004008b0002" \ + "003c0001000500010003001e010000050001008f000400a000080131000401320004013000020100" \ + "001a00000a5900000171352e67210000000100000000000104000054976500dfac110002ff7ed688" \ + "ff7ed73a000015c70000000d000000000000000001bbe1a6061b0400ac110002976500dfff7ed688" \ + "ff7ed73a0000074f000000130000000000000000e1a601bb061f04000401004cac110002ac110001" \ + "ff7db9e0ff7dc1d0000000fc00000003000000000000000008000400ac110001ac110002ff7db9e0" \ + "ff7dc1d0000000fc0000000300000000000000000000040008010220fde66f14e0f1960900000242" \ + "ac110002ff0200000000000000000001ff110001ff7dfad6ff7e0e95000001b00000000600000000" \ + "0000000087000600fde66f14e0f1960900000242ac110002fde66f14e0f196090000000000000001" \ + "ff7e567fff7e664a0000020800000005000000000000000080000600fde66f14e0f1960900000000" \ + "00000001fde66f14e0f1960900000242ac110002ff7e567fff7e664a000002080000000500000000" \ + "0000000081000600fe800000000000000042aafffe73bbfafde66f14e0f1960900000242ac110002" \ + "ff7e6aaaff7e6aaa0000004800000001000000000000000087000600fde66f14e0f1960900000242" \ + "ac110002fe800000000000000042aafffe73bbfaff7e6aaaff7e6aaa000000400000000100000000" \ + "0000000088000600fe800000000000000042acfffe110002fe800000000000000042aafffe73bbfa" \ + "ff7e7eaaff7e7eaa0000004800000001000000000000000087000600fe800000000000000042aaff" \ + "fe73bbfafe800000000000000042acfffe110002ff7e7eaaff7e7eaa000000400000000100000000" \ + "0000000088000600fe800000000000000042aafffe73bbfafe800000000000000042acfffe110002" \ + "ff7e92aaff7e92aa0000004800000001000000000000000087000600fe800000000000000042acff" \ + "fe110002fe800000000000000042aafffe73bbfaff7e92aaff7e92aa000000400000000100000000" \ + "000000008800060008000044fde66f14e0f1960900000242ac110002fd41b7143f86000000000000" \ + "00000001ff7ec2a0ff7ec2a00000004a000000010000000000000000d20100351100060004000054" \ + "ac1100027f000001ff7ed62eff7ed68700000036000000010000000000000000c496003511000400" \ + "7f000001ac110002ff7ed62eff7ed687000000760000000100000000000000000035c49611000400" \ + "08000044fde66f14e0f1960900000242ac110002fd41b7143f8600000000000000000001ff7ef359" \ + "ff7ef3590000004a000000010000000000000000b1e700351100060004000054ac1100027f000001" \ + "ff7f06e4ff7f06e800000036000000010000000000000000a8f90035110004007f000001ac110002" \ + "ff7f06e4ff7f06e8000000a60000000100000000000000000035a8f911000400" + +# Example export for IPFIX with two data sets +PACKET_IPFIX = "000a00d05e8465fd00000016000000000801007cfe800000000000000042acfffe110002fde66f14" \ + "e0f196090000000000000001ff7f0755ff7f07550000004800000001000000000000000087000600" \ + "fde66f14e0f196090000000000000001fe800000000000000042acfffe110002ff7f0755ff7f0755" \ + "000000400000000100000000000000008800060008000044fde66f14e0f1960900000242ac110002" \ + "2a044e42020000000000000000000223ff7f06e9ff7f22d500000140000000040000000000000000" \ + "e54c01bb06020600" + + class TestFlowExportIPFIX(unittest.TestCase): - pass + """Test IPFIX packet parsing + """ + def test_recv_ipfix_packet(self): + # send packet without any template, must fail to parse (packets are queued) + pkts, _, _ = send_recv_packets([PACKET_IPFIX]) + self.assertEqual(len(pkts), 0) # no export is parsed due to missing template + + # send packet with 5 templates and 20 flows, should parse correctly since the templates are known + pkts, _, _ = send_recv_packets([PACKET_IPFIX_TEMPLATE]) + self.assertEqual(len(pkts), 1) + p = pkts[0] + self.assertEqual(p.client[0], "127.0.0.1") + self.assertEqual(len(p.export.flows), 1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) # count flows + self.assertEqual(len(p.export.templates), 4 + 1) # count new templates + + # Inspect contents of specific flows + flow = p.export.flows[0] + self.assertEqual(flow.meteringProcessId, 2649) + self.assertEqual(flow.selectorAlgorithm, 1) + self.assertEqual(flow.systemInitTimeMilliseconds, 1585735165729) + + flow = p.export.flows[1] # HTTPS flow from web server to client + self.assertEqual(flow.destinationIPv4Address, 2886795266) + self.assertEqual(ipaddress.ip_address(flow.destinationIPv4Address), + ipaddress.ip_address("172.17.0.2")) + self.assertEqual(flow.protocolIdentifier, 6) # TCP + self.assertEqual(flow.sourceTransportPort, 443) + self.assertEqual(flow.destinationTransportPort, 57766) + + flow = p.export.flows[17] # IPv6 flow + self.assertEqual(flow.protocolIdentifier, 17) # UDP + self.assertEqual(flow.sourceIPv6Address, 337491164212692683663430561043420610562) + self.assertEqual(ipaddress.ip_address(flow.sourceIPv6Address), # Docker ULA + ipaddress.ip_address("fde6:6f14:e0f1:9609:0:242:ac11:2")) + + # send template and multiple export packets + pkts, _, _ = send_recv_packets([PACKET_IPFIX, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX]) + self.assertEqual(len(pkts), 3) + self.assertEqual(pkts[0].export.header.version, 10) + + # check amount of flows across all packets + total_flows = 0 + for packet in pkts: + total_flows += len(packet.export.flows) + self.assertEqual(total_flows, 2 + 1 + (1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) + 2 + 1) diff --git a/tests/test_netflow.py b/tests/test_netflow.py index 2673f33..a820362 100755 --- a/tests/test_netflow.py +++ b/tests/test_netflow.py @@ -15,6 +15,94 @@ import unittest from tests.lib import * +# Example export for v1 which contains two flows from one ICMP ping request/reply session +PACKET_V1 = "000100020001189b5e80c32c2fd41848ac110002ac11000100000000000000000000000a00000348" \ + "000027c700004af100000800000001000000000000000000ac110001ac1100020000000000000000" \ + "0000000a00000348000027c700004af100000000000001000000000000000000" + +# Example export for v5 which contains three flows, two for ICMP ping and one multicast on interface (224.0.0.251) +PACKET_V5 = "00050003000379a35e80c58622a55ab00000000000000000ac110002ac1100010000000000000000" \ + "0000000a0000034800002f4c0000527600000800000001000000000000000000ac110001ac110002" \ + "00000000000000000000000a0000034800002f4c0000527600000000000001000000000000000000" \ + "ac110001e00000fb000000000000000000000001000000a90000e01c0000e01c14e914e900001100" \ + "0000000000000000" + +PACKET_V9_TEMPLATE = "0009000a000000035c9f55980000000100000000000000400400000e00080004000c000400150004" \ + "001600040001000400020004000a0004000e000400070002000b00020004000100060001003c0001" \ + "00050001000000400800000e001b0010001c001000150004001600040001000400020004000a0004" \ + "000e000400070002000b00020004000100060001003c000100050001040001447f0000017f000001" \ + "fb3c1aaafb3c18fd000190100000004b00000000000000000050942c061b04007f0000017f000001" \ + "fb3c1aaafb3c18fd00000f94000000360000000000000000942c0050061f04007f0000017f000001" \ + "fb3c1cfcfb3c1a9b0000d3fc0000002a000000000000000000509434061b04007f0000017f000001" \ + "fb3c1cfcfb3c1a9b00000a490000001e000000000000000094340050061f04007f0000017f000001" \ + "fb3bb82cfb3ba48b000002960000000300000000000000000050942a061904007f0000017f000001" \ + "fb3bb82cfb3ba48b00000068000000020000000000000000942a0050061104007f0000017f000001" \ + "fb3c1900fb3c18fe0000004c0000000100000000000000000035b3c9110004007f0000017f000001" \ + "fb3c1900fb3c18fe0000003c000000010000000000000000b3c9003511000400" + +# This packet is special. We take PACKET_V9_TEMPLATE and re-order the templates and flows. +# The first line is the header, the smaller lines the templates and the long lines the flows (limited to 80 chars) +PACKET_V9_TEMPLATE_MIXED = ("0009000a000000035c9f55980000000100000000" # header + "040001447f0000017f000001fb3c1aaafb3c18fd000190100000004b00000000000000000050942c" + "061b04007f0000017f000001fb3c1aaafb3c18fd00000f94000000360000000000000000942c0050" + "061f04007f0000017f000001fb3c1cfcfb3c1a9b0000d3fc0000002a000000000000000000509434" + "061b04007f0000017f000001fb3c1cfcfb3c1a9b00000a490000001e000000000000000094340050" + "061f04007f0000017f000001fb3bb82cfb3ba48b000002960000000300000000000000000050942a" + "061904007f0000017f000001fb3bb82cfb3ba48b00000068000000020000000000000000942a0050" + "061104007f0000017f000001fb3c1900fb3c18fe0000004c0000000100000000000000000035b3c9" + "110004007f0000017f000001fb3c1900fb3c18fe0000003c000000010000000000000000b3c90035" + "11000400" # end of flow segments + "000000400400000e00080004000c000400150004001600040001000400020004" # template 1024 + "000a0004000e000400070002000b00020004000100060001003c000100050001" + "000000400800000e001b0010001c001000150004001600040001000400020004" # template 2048 + "000a0004000e000400070002000b00020004000100060001003c000100050001") + +# Three packets without templates, each with 12 flows, anonymized +PACKETS_V9 = [ + "0009000c000000035c9f55980000000200000000040001e47f0000017f000001fb3c1a17fb3c19fd" + "000001480000000200000000000000000035ea82110004007f0000017f000001fb3c1a17fb3c19fd" + "0000007a000000020000000000000000ea820035110004007f0000017f000001fb3c1a17fb3c19fd" + "000000f80000000200000000000000000035c6e2110004007f0000017f000001fb3c1a17fb3c19fd" + "0000007a000000020000000000000000c6e20035110004007f0000017f000001fb3c1a9efb3c1a9c" + "0000004c0000000100000000000000000035adc1110004007f0000017f000001fb3c1a9efb3c1a9c" + "0000003c000000010000000000000000adc10035110004007f0000017f000001fb3c1b74fb3c1b72" + "0000004c0000000100000000000000000035d0b3110004007f0000017f000001fb3c1b74fb3c1b72" + "0000003c000000010000000000000000d0b30035110004007f0000017f000001fb3c2f59fb3c1b71" + "00001a350000000a000000000000000000509436061b04007f0000017f000001fb3c2f59fb3c1b71" + "0000038a0000000a000000000000000094360050061b04007f0000017f000001fb3c913bfb3c9138" + "0000004c0000000100000000000000000035e262110004007f0000017f000001fb3c913bfb3c9138" + "0000003c000000010000000000000000e262003511000400", + + "0009000c000000035c9f55980000000300000000040001e47f0000017f000001fb3ca523fb3c913b" + "0000030700000005000000000000000000509438061b04007f0000017f000001fb3ca523fb3c913b" + "000002a200000005000000000000000094380050061b04007f0000017f000001fb3f7fe1fb3dbc97" + "0002d52800000097000000000000000001bb8730061b04007f0000017f000001fb3f7fe1fb3dbc97" + "0000146c000000520000000000000000873001bb061f04007f0000017f000001fb3d066ffb3d066c" + "0000004c0000000100000000000000000035e5bd110004007f0000017f000001fb3d066ffb3d066c" + "0000003c000000010000000000000000e5bd0035110004007f0000017f000001fb3d1a61fb3d066b" + "000003060000000500000000000000000050943a061b04007f0000017f000001fb3d1a61fb3d066b" + "000002a2000000050000000000000000943a0050061b04007f0000017f000001fb3fed00fb3f002c" + "0000344000000016000000000000000001bbae50061f04007f0000017f000001fb3fed00fb3f002c" + "00000a47000000120000000000000000ae5001bb061b04007f0000017f000001fb402f17fb402a75" + "0003524c000000a5000000000000000001bbc48c061b04007f0000017f000001fb402f17fb402a75" + "000020a60000007e0000000000000000c48c01bb061f0400", + + "0009000c000000035c9f55980000000400000000040001e47f0000017f000001fb3d7ba2fb3d7ba0" + "0000004c0000000100000000000000000035a399110004007f0000017f000001fb3d7ba2fb3d7ba0" + "0000003c000000010000000000000000a3990035110004007f0000017f000001fb3d8f85fb3d7b9f" + "000003070000000500000000000000000050943c061b04007f0000017f000001fb3d8f85fb3d7b9f" + "000002a2000000050000000000000000943c0050061b04007f0000017f000001fb3d9165fb3d7f6d" + "0000c97b0000002a000000000000000001bbae48061b04007f0000017f000001fb3d9165fb3d7f6d" + "000007f40000001a0000000000000000ae4801bb061b04007f0000017f000001fb3dbc96fb3dbc7e" + "0000011e0000000200000000000000000035bd4f110004007f0000017f000001fb3dbc96fb3dbc7e" + "0000008e000000020000000000000000bd4f0035110004007f0000017f000001fb3ddbb3fb3c1a18" + "0000bfee0000002f00000000000000000050ae56061b04007f0000017f000001fb3ddbb3fb3c1a18" + "00000982000000270000000000000000ae560050061b04007f0000017f000001fb3ddbb3fb3c1a18" + "0000130e0000001200000000000000000050e820061b04007f0000017f000001fb3ddbb3fb3c1a18" + "0000059c000000140000000000000000e8200050061b0400" +] + + class TestFlowExportNetflow(unittest.TestCase): def _test_recv_all_packets(self, num, template_idx, delay=0.0001): """Fling packets at the server and test that it receives them all"""