diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/lib.py b/tests/lib.py index f769460..041831a 100644 --- a/tests/lib.py +++ b/tests/lib.py @@ -15,92 +15,6 @@ import time from netflow.collector import ThreadedNetFlowListener -PACKET_V9_TEMPLATE = "0009000a000000035c9f55980000000100000000000000400400000e00080004000c000400150004" \ - "001600040001000400020004000a0004000e000400070002000b00020004000100060001003c0001" \ - "00050001000000400800000e001b0010001c001000150004001600040001000400020004000a0004" \ - "000e000400070002000b00020004000100060001003c000100050001040001447f0000017f000001" \ - "fb3c1aaafb3c18fd000190100000004b00000000000000000050942c061b04007f0000017f000001" \ - "fb3c1aaafb3c18fd00000f94000000360000000000000000942c0050061f04007f0000017f000001" \ - "fb3c1cfcfb3c1a9b0000d3fc0000002a000000000000000000509434061b04007f0000017f000001" \ - "fb3c1cfcfb3c1a9b00000a490000001e000000000000000094340050061f04007f0000017f000001" \ - "fb3bb82cfb3ba48b000002960000000300000000000000000050942a061904007f0000017f000001" \ - "fb3bb82cfb3ba48b00000068000000020000000000000000942a0050061104007f0000017f000001" \ - "fb3c1900fb3c18fe0000004c0000000100000000000000000035b3c9110004007f0000017f000001" \ - "fb3c1900fb3c18fe0000003c000000010000000000000000b3c9003511000400" - -# This packet is special. We take PACKET_V9_TEMPLATE and re-order the templates and flows. -# The first line is the header, the smaller lines the templates and the long lines the flows (limited to 80 chars) -PACKET_V9_TEMPLATE_MIXED = ("0009000a000000035c9f55980000000100000000" # header - "040001447f0000017f000001fb3c1aaafb3c18fd000190100000004b00000000000000000050942c" - "061b04007f0000017f000001fb3c1aaafb3c18fd00000f94000000360000000000000000942c0050" - "061f04007f0000017f000001fb3c1cfcfb3c1a9b0000d3fc0000002a000000000000000000509434" - "061b04007f0000017f000001fb3c1cfcfb3c1a9b00000a490000001e000000000000000094340050" - "061f04007f0000017f000001fb3bb82cfb3ba48b000002960000000300000000000000000050942a" - "061904007f0000017f000001fb3bb82cfb3ba48b00000068000000020000000000000000942a0050" - "061104007f0000017f000001fb3c1900fb3c18fe0000004c0000000100000000000000000035b3c9" - "110004007f0000017f000001fb3c1900fb3c18fe0000003c000000010000000000000000b3c90035" - "11000400" # end of flow segments - "000000400400000e00080004000c000400150004001600040001000400020004" # template 1024 - "000a0004000e000400070002000b00020004000100060001003c000100050001" - "000000400800000e001b0010001c001000150004001600040001000400020004" # template 2048 - "000a0004000e000400070002000b00020004000100060001003c000100050001") - -# Three packets without templates, each with 12 flows, anonymized -PACKETS_V9 = [ - "0009000c000000035c9f55980000000200000000040001e47f0000017f000001fb3c1a17fb3c19fd" - "000001480000000200000000000000000035ea82110004007f0000017f000001fb3c1a17fb3c19fd" - "0000007a000000020000000000000000ea820035110004007f0000017f000001fb3c1a17fb3c19fd" - "000000f80000000200000000000000000035c6e2110004007f0000017f000001fb3c1a17fb3c19fd" - "0000007a000000020000000000000000c6e20035110004007f0000017f000001fb3c1a9efb3c1a9c" - "0000004c0000000100000000000000000035adc1110004007f0000017f000001fb3c1a9efb3c1a9c" - "0000003c000000010000000000000000adc10035110004007f0000017f000001fb3c1b74fb3c1b72" - "0000004c0000000100000000000000000035d0b3110004007f0000017f000001fb3c1b74fb3c1b72" - "0000003c000000010000000000000000d0b30035110004007f0000017f000001fb3c2f59fb3c1b71" - "00001a350000000a000000000000000000509436061b04007f0000017f000001fb3c2f59fb3c1b71" - "0000038a0000000a000000000000000094360050061b04007f0000017f000001fb3c913bfb3c9138" - "0000004c0000000100000000000000000035e262110004007f0000017f000001fb3c913bfb3c9138" - "0000003c000000010000000000000000e262003511000400", - - "0009000c000000035c9f55980000000300000000040001e47f0000017f000001fb3ca523fb3c913b" - "0000030700000005000000000000000000509438061b04007f0000017f000001fb3ca523fb3c913b" - "000002a200000005000000000000000094380050061b04007f0000017f000001fb3f7fe1fb3dbc97" - "0002d52800000097000000000000000001bb8730061b04007f0000017f000001fb3f7fe1fb3dbc97" - "0000146c000000520000000000000000873001bb061f04007f0000017f000001fb3d066ffb3d066c" - "0000004c0000000100000000000000000035e5bd110004007f0000017f000001fb3d066ffb3d066c" - "0000003c000000010000000000000000e5bd0035110004007f0000017f000001fb3d1a61fb3d066b" - "000003060000000500000000000000000050943a061b04007f0000017f000001fb3d1a61fb3d066b" - "000002a2000000050000000000000000943a0050061b04007f0000017f000001fb3fed00fb3f002c" - "0000344000000016000000000000000001bbae50061f04007f0000017f000001fb3fed00fb3f002c" - "00000a47000000120000000000000000ae5001bb061b04007f0000017f000001fb402f17fb402a75" - "0003524c000000a5000000000000000001bbc48c061b04007f0000017f000001fb402f17fb402a75" - "000020a60000007e0000000000000000c48c01bb061f0400", - - "0009000c000000035c9f55980000000400000000040001e47f0000017f000001fb3d7ba2fb3d7ba0" - "0000004c0000000100000000000000000035a399110004007f0000017f000001fb3d7ba2fb3d7ba0" - "0000003c000000010000000000000000a3990035110004007f0000017f000001fb3d8f85fb3d7b9f" - "000003070000000500000000000000000050943c061b04007f0000017f000001fb3d8f85fb3d7b9f" - "000002a2000000050000000000000000943c0050061b04007f0000017f000001fb3d9165fb3d7f6d" - "0000c97b0000002a000000000000000001bbae48061b04007f0000017f000001fb3d9165fb3d7f6d" - "000007f40000001a0000000000000000ae4801bb061b04007f0000017f000001fb3dbc96fb3dbc7e" - "0000011e0000000200000000000000000035bd4f110004007f0000017f000001fb3dbc96fb3dbc7e" - "0000008e000000020000000000000000bd4f0035110004007f0000017f000001fb3ddbb3fb3c1a18" - "0000bfee0000002f00000000000000000050ae56061b04007f0000017f000001fb3ddbb3fb3c1a18" - "00000982000000270000000000000000ae560050061b04007f0000017f000001fb3ddbb3fb3c1a18" - "0000130e0000001200000000000000000050e820061b04007f0000017f000001fb3ddbb3fb3c1a18" - "0000059c000000140000000000000000e8200050061b0400" -] - -# Example export for v1 which contains two flows from one ICMP ping request/reply session -PACKET_V1 = "000100020001189b5e80c32c2fd41848ac110002ac11000100000000000000000000000a00000348" \ - "000027c700004af100000800000001000000000000000000ac110001ac1100020000000000000000" \ - "0000000a00000348000027c700004af100000000000001000000000000000000" - -# Example export for v5 which contains three flows, two for ICMP ping and one multicast on interface (224.0.0.251) -PACKET_V5 = "00050003000379a35e80c58622a55ab00000000000000000ac110002ac1100010000000000000000" \ - "0000000a0000034800002f4c0000527600000800000001000000000000000000ac110001ac110002" \ - "00000000000000000000000a0000034800002f4c0000527600000000000001000000000000000000" \ - "ac110001e00000fb000000000000000000000001000000a90000e01c0000e01c14e914e900001100" \ - "0000000000000000" # Invalid export hex stream PACKET_INVALID = "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" diff --git a/tests/test_ipfix.py b/tests/test_ipfix.py index 06afa7e..db4ab3d 100644 --- a/tests/test_ipfix.py +++ b/tests/test_ipfix.py @@ -9,11 +9,98 @@ Licensed under MIT License. See LICENSE. # TODO: tests with 500 packets fail with delay=0. Probably a problem with UDP sockets buffer import ipaddress -import random import unittest from tests.lib import * +# Example export for IPFIX (v10) with 4 templates, 1 option template and 8 data flow sets +PACKET_IPFIX_TEMPLATE = "000a05205e8465fd0000001300000000000200400400000e00080004000c00040016000400150004" \ + "0001000400020004000a0004000e000400070002000b00020004000100060001003c000100050001" \ + "000200340401000b00080004000c000400160004001500040001000400020004000a0004000e0004" \ + "00200002003c000100050001000200400800000e001b0010001c0010001600040015000400010004" \ + "00020004000a0004000e000400070002000b00020004000100060001003c00010005000100020034" \ + "0801000b001b0010001c001000160004001500040001000400020004000a0004000e0004008b0002" \ + "003c0001000500010003001e010000050001008f000400a000080131000401320004013000020100" \ + "001a00000a5900000171352e67210000000100000000000104000054976500dfac110002ff7ed688" \ + "ff7ed73a000015c70000000d000000000000000001bbe1a6061b0400ac110002976500dfff7ed688" \ + "ff7ed73a0000074f000000130000000000000000e1a601bb061f04000401004cac110002ac110001" \ + "ff7db9e0ff7dc1d0000000fc00000003000000000000000008000400ac110001ac110002ff7db9e0" \ + "ff7dc1d0000000fc0000000300000000000000000000040008010220fde66f14e0f1960900000242" \ + "ac110002ff0200000000000000000001ff110001ff7dfad6ff7e0e95000001b00000000600000000" \ + "0000000087000600fde66f14e0f1960900000242ac110002fde66f14e0f196090000000000000001" \ + "ff7e567fff7e664a0000020800000005000000000000000080000600fde66f14e0f1960900000000" \ + "00000001fde66f14e0f1960900000242ac110002ff7e567fff7e664a000002080000000500000000" \ + "0000000081000600fe800000000000000042aafffe73bbfafde66f14e0f1960900000242ac110002" \ + "ff7e6aaaff7e6aaa0000004800000001000000000000000087000600fde66f14e0f1960900000242" \ + "ac110002fe800000000000000042aafffe73bbfaff7e6aaaff7e6aaa000000400000000100000000" \ + "0000000088000600fe800000000000000042acfffe110002fe800000000000000042aafffe73bbfa" \ + "ff7e7eaaff7e7eaa0000004800000001000000000000000087000600fe800000000000000042aaff" \ + "fe73bbfafe800000000000000042acfffe110002ff7e7eaaff7e7eaa000000400000000100000000" \ + "0000000088000600fe800000000000000042aafffe73bbfafe800000000000000042acfffe110002" \ + "ff7e92aaff7e92aa0000004800000001000000000000000087000600fe800000000000000042acff" \ + "fe110002fe800000000000000042aafffe73bbfaff7e92aaff7e92aa000000400000000100000000" \ + "000000008800060008000044fde66f14e0f1960900000242ac110002fd41b7143f86000000000000" \ + "00000001ff7ec2a0ff7ec2a00000004a000000010000000000000000d20100351100060004000054" \ + "ac1100027f000001ff7ed62eff7ed68700000036000000010000000000000000c496003511000400" \ + "7f000001ac110002ff7ed62eff7ed687000000760000000100000000000000000035c49611000400" \ + "08000044fde66f14e0f1960900000242ac110002fd41b7143f8600000000000000000001ff7ef359" \ + "ff7ef3590000004a000000010000000000000000b1e700351100060004000054ac1100027f000001" \ + "ff7f06e4ff7f06e800000036000000010000000000000000a8f90035110004007f000001ac110002" \ + "ff7f06e4ff7f06e8000000a60000000100000000000000000035a8f911000400" + +# Example export for IPFIX with two data sets +PACKET_IPFIX = "000a00d05e8465fd00000016000000000801007cfe800000000000000042acfffe110002fde66f14" \ + "e0f196090000000000000001ff7f0755ff7f07550000004800000001000000000000000087000600" \ + "fde66f14e0f196090000000000000001fe800000000000000042acfffe110002ff7f0755ff7f0755" \ + "000000400000000100000000000000008800060008000044fde66f14e0f1960900000242ac110002" \ + "2a044e42020000000000000000000223ff7f06e9ff7f22d500000140000000040000000000000000" \ + "e54c01bb06020600" + + class TestFlowExportIPFIX(unittest.TestCase): - pass + """Test IPFIX packet parsing + """ + def test_recv_ipfix_packet(self): + # send packet without any template, must fail to parse (packets are queued) + pkts, _, _ = send_recv_packets([PACKET_IPFIX]) + self.assertEqual(len(pkts), 0) # no export is parsed due to missing template + + # send packet with 5 templates and 20 flows, should parse correctly since the templates are known + pkts, _, _ = send_recv_packets([PACKET_IPFIX_TEMPLATE]) + self.assertEqual(len(pkts), 1) + p = pkts[0] + self.assertEqual(p.client[0], "127.0.0.1") + self.assertEqual(len(p.export.flows), 1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) # count flows + self.assertEqual(len(p.export.templates), 4 + 1) # count new templates + + # Inspect contents of specific flows + flow = p.export.flows[0] + self.assertEqual(flow.meteringProcessId, 2649) + self.assertEqual(flow.selectorAlgorithm, 1) + self.assertEqual(flow.systemInitTimeMilliseconds, 1585735165729) + + flow = p.export.flows[1] # HTTPS flow from web server to client + self.assertEqual(flow.destinationIPv4Address, 2886795266) + self.assertEqual(ipaddress.ip_address(flow.destinationIPv4Address), + ipaddress.ip_address("172.17.0.2")) + self.assertEqual(flow.protocolIdentifier, 6) # TCP + self.assertEqual(flow.sourceTransportPort, 443) + self.assertEqual(flow.destinationTransportPort, 57766) + + flow = p.export.flows[17] # IPv6 flow + self.assertEqual(flow.protocolIdentifier, 17) # UDP + self.assertEqual(flow.sourceIPv6Address, 337491164212692683663430561043420610562) + self.assertEqual(ipaddress.ip_address(flow.sourceIPv6Address), # Docker ULA + ipaddress.ip_address("fde6:6f14:e0f1:9609:0:242:ac11:2")) + + # send template and multiple export packets + pkts, _, _ = send_recv_packets([PACKET_IPFIX, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX]) + self.assertEqual(len(pkts), 3) + self.assertEqual(pkts[0].export.header.version, 10) + + # check amount of flows across all packets + total_flows = 0 + for packet in pkts: + total_flows += len(packet.export.flows) + self.assertEqual(total_flows, 2 + 1 + (1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) + 2 + 1) diff --git a/tests/test_netflow.py b/tests/test_netflow.py index 2673f33..a820362 100755 --- a/tests/test_netflow.py +++ b/tests/test_netflow.py @@ -15,6 +15,94 @@ import unittest from tests.lib import * +# Example export for v1 which contains two flows from one ICMP ping request/reply session +PACKET_V1 = "000100020001189b5e80c32c2fd41848ac110002ac11000100000000000000000000000a00000348" \ + "000027c700004af100000800000001000000000000000000ac110001ac1100020000000000000000" \ + "0000000a00000348000027c700004af100000000000001000000000000000000" + +# Example export for v5 which contains three flows, two for ICMP ping and one multicast on interface (224.0.0.251) +PACKET_V5 = "00050003000379a35e80c58622a55ab00000000000000000ac110002ac1100010000000000000000" \ + "0000000a0000034800002f4c0000527600000800000001000000000000000000ac110001ac110002" \ + "00000000000000000000000a0000034800002f4c0000527600000000000001000000000000000000" \ + "ac110001e00000fb000000000000000000000001000000a90000e01c0000e01c14e914e900001100" \ + "0000000000000000" + +PACKET_V9_TEMPLATE = "0009000a000000035c9f55980000000100000000000000400400000e00080004000c000400150004" \ + "001600040001000400020004000a0004000e000400070002000b00020004000100060001003c0001" \ + "00050001000000400800000e001b0010001c001000150004001600040001000400020004000a0004" \ + "000e000400070002000b00020004000100060001003c000100050001040001447f0000017f000001" \ + "fb3c1aaafb3c18fd000190100000004b00000000000000000050942c061b04007f0000017f000001" \ + "fb3c1aaafb3c18fd00000f94000000360000000000000000942c0050061f04007f0000017f000001" \ + "fb3c1cfcfb3c1a9b0000d3fc0000002a000000000000000000509434061b04007f0000017f000001" \ + "fb3c1cfcfb3c1a9b00000a490000001e000000000000000094340050061f04007f0000017f000001" \ + "fb3bb82cfb3ba48b000002960000000300000000000000000050942a061904007f0000017f000001" \ + "fb3bb82cfb3ba48b00000068000000020000000000000000942a0050061104007f0000017f000001" \ + "fb3c1900fb3c18fe0000004c0000000100000000000000000035b3c9110004007f0000017f000001" \ + "fb3c1900fb3c18fe0000003c000000010000000000000000b3c9003511000400" + +# This packet is special. We take PACKET_V9_TEMPLATE and re-order the templates and flows. +# The first line is the header, the smaller lines the templates and the long lines the flows (limited to 80 chars) +PACKET_V9_TEMPLATE_MIXED = ("0009000a000000035c9f55980000000100000000" # header + "040001447f0000017f000001fb3c1aaafb3c18fd000190100000004b00000000000000000050942c" + "061b04007f0000017f000001fb3c1aaafb3c18fd00000f94000000360000000000000000942c0050" + "061f04007f0000017f000001fb3c1cfcfb3c1a9b0000d3fc0000002a000000000000000000509434" + "061b04007f0000017f000001fb3c1cfcfb3c1a9b00000a490000001e000000000000000094340050" + "061f04007f0000017f000001fb3bb82cfb3ba48b000002960000000300000000000000000050942a" + "061904007f0000017f000001fb3bb82cfb3ba48b00000068000000020000000000000000942a0050" + "061104007f0000017f000001fb3c1900fb3c18fe0000004c0000000100000000000000000035b3c9" + "110004007f0000017f000001fb3c1900fb3c18fe0000003c000000010000000000000000b3c90035" + "11000400" # end of flow segments + "000000400400000e00080004000c000400150004001600040001000400020004" # template 1024 + "000a0004000e000400070002000b00020004000100060001003c000100050001" + "000000400800000e001b0010001c001000150004001600040001000400020004" # template 2048 + "000a0004000e000400070002000b00020004000100060001003c000100050001") + +# Three packets without templates, each with 12 flows, anonymized +PACKETS_V9 = [ + "0009000c000000035c9f55980000000200000000040001e47f0000017f000001fb3c1a17fb3c19fd" + "000001480000000200000000000000000035ea82110004007f0000017f000001fb3c1a17fb3c19fd" + "0000007a000000020000000000000000ea820035110004007f0000017f000001fb3c1a17fb3c19fd" + "000000f80000000200000000000000000035c6e2110004007f0000017f000001fb3c1a17fb3c19fd" + "0000007a000000020000000000000000c6e20035110004007f0000017f000001fb3c1a9efb3c1a9c" + "0000004c0000000100000000000000000035adc1110004007f0000017f000001fb3c1a9efb3c1a9c" + "0000003c000000010000000000000000adc10035110004007f0000017f000001fb3c1b74fb3c1b72" + "0000004c0000000100000000000000000035d0b3110004007f0000017f000001fb3c1b74fb3c1b72" + "0000003c000000010000000000000000d0b30035110004007f0000017f000001fb3c2f59fb3c1b71" + "00001a350000000a000000000000000000509436061b04007f0000017f000001fb3c2f59fb3c1b71" + "0000038a0000000a000000000000000094360050061b04007f0000017f000001fb3c913bfb3c9138" + "0000004c0000000100000000000000000035e262110004007f0000017f000001fb3c913bfb3c9138" + "0000003c000000010000000000000000e262003511000400", + + "0009000c000000035c9f55980000000300000000040001e47f0000017f000001fb3ca523fb3c913b" + "0000030700000005000000000000000000509438061b04007f0000017f000001fb3ca523fb3c913b" + "000002a200000005000000000000000094380050061b04007f0000017f000001fb3f7fe1fb3dbc97" + "0002d52800000097000000000000000001bb8730061b04007f0000017f000001fb3f7fe1fb3dbc97" + "0000146c000000520000000000000000873001bb061f04007f0000017f000001fb3d066ffb3d066c" + "0000004c0000000100000000000000000035e5bd110004007f0000017f000001fb3d066ffb3d066c" + "0000003c000000010000000000000000e5bd0035110004007f0000017f000001fb3d1a61fb3d066b" + "000003060000000500000000000000000050943a061b04007f0000017f000001fb3d1a61fb3d066b" + "000002a2000000050000000000000000943a0050061b04007f0000017f000001fb3fed00fb3f002c" + "0000344000000016000000000000000001bbae50061f04007f0000017f000001fb3fed00fb3f002c" + "00000a47000000120000000000000000ae5001bb061b04007f0000017f000001fb402f17fb402a75" + "0003524c000000a5000000000000000001bbc48c061b04007f0000017f000001fb402f17fb402a75" + "000020a60000007e0000000000000000c48c01bb061f0400", + + "0009000c000000035c9f55980000000400000000040001e47f0000017f000001fb3d7ba2fb3d7ba0" + "0000004c0000000100000000000000000035a399110004007f0000017f000001fb3d7ba2fb3d7ba0" + "0000003c000000010000000000000000a3990035110004007f0000017f000001fb3d8f85fb3d7b9f" + "000003070000000500000000000000000050943c061b04007f0000017f000001fb3d8f85fb3d7b9f" + "000002a2000000050000000000000000943c0050061b04007f0000017f000001fb3d9165fb3d7f6d" + "0000c97b0000002a000000000000000001bbae48061b04007f0000017f000001fb3d9165fb3d7f6d" + "000007f40000001a0000000000000000ae4801bb061b04007f0000017f000001fb3dbc96fb3dbc7e" + "0000011e0000000200000000000000000035bd4f110004007f0000017f000001fb3dbc96fb3dbc7e" + "0000008e000000020000000000000000bd4f0035110004007f0000017f000001fb3ddbb3fb3c1a18" + "0000bfee0000002f00000000000000000050ae56061b04007f0000017f000001fb3ddbb3fb3c1a18" + "00000982000000270000000000000000ae560050061b04007f0000017f000001fb3ddbb3fb3c1a18" + "0000130e0000001200000000000000000050e820061b04007f0000017f000001fb3ddbb3fb3c1a18" + "0000059c000000140000000000000000e8200050061b0400" +] + + class TestFlowExportNetflow(unittest.TestCase): def _test_recv_all_packets(self, num, template_idx, delay=0.0001): """Fling packets at the server and test that it receives them all"""