959f8d3c2c
The function send_recv_packets in tests stored all processed ExportPackets by default in a list. Memory usage tests were therefore based on this high amount of stored objects, since no instance of any ExportPacket was deleted until exit. With the new parameter store_packets the caller can define how many packets should be stored during receiving, as to test multiple scenarios. Three such scenarios are implemented: don't store any packet, store maximum of 500 at a time and store all packets. This comes much closer to the real world scenario of the collector, which uses a "for export in listener.get" loop, dumping any new ExportPacket to file immediatelly and then deleting the object. Yet, the case where all packets are stored must still be covered as well, because the collector might not be the only implementation which uses listener.get, so finding memory leaks should be covered.
232 lines
14 KiB
Python
232 lines
14 KiB
Python
"""
|
|
This file belongs to https://github.com/bitkeks/python-netflow-v9-softflowd.
|
|
|
|
The test packets (defined below as hex streams) were extracted from "real"
|
|
softflowd exports based on a sample PCAP capture file.
|
|
|
|
Copyright 2016-2020 Dominik Pataky <dev@bitkeks.eu>
|
|
Licensed under MIT License. See LICENSE.
|
|
"""
|
|
|
|
# The flowset with 2 templates (IPv4 and IPv6) and 8 flows with data
|
|
import queue
|
|
import random
|
|
import socket
|
|
import time
|
|
|
|
from netflow.collector import ThreadedNetFlowListener
|
|
|
|
# Invalid export hex stream
|
|
PACKET_INVALID = "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"
|
|
|
|
CONNECTION = ('127.0.0.1', 1337)
|
|
NUM_PACKETS = 1000
|
|
|
|
|
|
def emit_packets(packets, delay=0.0001):
|
|
"""Send the provided packets to the listener"""
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
for p in packets:
|
|
sock.sendto(bytes.fromhex(p), CONNECTION)
|
|
time.sleep(delay)
|
|
sock.close()
|
|
|
|
|
|
def send_recv_packets(packets, delay=0.0001, store_packets=-1) -> (list, float, float):
|
|
"""Starts a listener, send packets, receives packets
|
|
|
|
returns a tuple: ([(ts, export), ...], time_started_sending, time_stopped_sending)
|
|
"""
|
|
listener = ThreadedNetFlowListener(*CONNECTION)
|
|
tstart = time.time()
|
|
emit_packets(packets, delay=delay)
|
|
time.sleep(0.5) # Allow packets to be sent and recieved
|
|
tend = time.time()
|
|
listener.start()
|
|
|
|
pkts = []
|
|
to_pad = 0
|
|
while True:
|
|
try:
|
|
packet = listener.get(timeout=0.5)
|
|
if -1 == store_packets or store_packets > 0:
|
|
# Case where a programm yields from the queue and stores all packets.
|
|
pkts.append(packet)
|
|
if store_packets != -1 and len(pkts) > store_packets:
|
|
to_pad += len(pkts) # Hack for testing
|
|
pkts.clear()
|
|
else:
|
|
# Performance measurements for cases where yielded objects are processed
|
|
# immediatelly instead of stored. Add empty tuple to retain counting possibility.
|
|
pkts.append(())
|
|
except queue.Empty:
|
|
break
|
|
listener.stop()
|
|
listener.join()
|
|
if to_pad > 0:
|
|
pkts = [()] * to_pad + pkts
|
|
return pkts, tstart, tend
|
|
|
|
|
|
def generate_packets(amount, version, template_every_x=100):
|
|
packets = [PACKET_IPFIX]
|
|
template = PACKET_IPFIX_TEMPLATE
|
|
|
|
if version == 1:
|
|
packets = [PACKET_V1]
|
|
elif version == 5:
|
|
packets = [PACKET_V5]
|
|
elif version == 9:
|
|
packets = [*PACKETS_V9]
|
|
template = PACKET_V9_TEMPLATE
|
|
|
|
if amount < template_every_x:
|
|
template_every_x = 10
|
|
|
|
# If the list of test packets is only one item big (the same packet is used over and over),
|
|
# do not use random.choice - it costs performance and results in the same packet every time.
|
|
def single_packet(pkts):
|
|
return pkts[0]
|
|
packet_func = single_packet
|
|
if len(packets) > 1:
|
|
packet_func = random.choice
|
|
|
|
for x in range(amount):
|
|
if x % template_every_x == 0 and version in [9, 10]:
|
|
# First packet always a template, then periodically
|
|
# Note: this was once based on random.random, but it costs performance
|
|
yield template
|
|
else:
|
|
yield packet_func(packets)
|
|
|
|
|
|
# Example export for v1 which contains two flows from one ICMP ping request/reply session
|
|
PACKET_V1 = "000100020001189b5e80c32c2fd41848ac110002ac11000100000000000000000000000a00000348" \
|
|
"000027c700004af100000800000001000000000000000000ac110001ac1100020000000000000000" \
|
|
"0000000a00000348000027c700004af100000000000001000000000000000000"
|
|
|
|
# Example export for v5 which contains three flows, two for ICMP ping and one multicast on interface (224.0.0.251)
|
|
PACKET_V5 = "00050003000379a35e80c58622a55ab00000000000000000ac110002ac1100010000000000000000" \
|
|
"0000000a0000034800002f4c0000527600000800000001000000000000000000ac110001ac110002" \
|
|
"00000000000000000000000a0000034800002f4c0000527600000000000001000000000000000000" \
|
|
"ac110001e00000fb000000000000000000000001000000a90000e01c0000e01c14e914e900001100" \
|
|
"0000000000000000"
|
|
|
|
PACKET_V9_TEMPLATE = "0009000a000000035c9f55980000000100000000000000400400000e00080004000c000400150004" \
|
|
"001600040001000400020004000a0004000e000400070002000b00020004000100060001003c0001" \
|
|
"00050001000000400800000e001b0010001c001000150004001600040001000400020004000a0004" \
|
|
"000e000400070002000b00020004000100060001003c000100050001040001447f0000017f000001" \
|
|
"fb3c1aaafb3c18fd000190100000004b00000000000000000050942c061b04007f0000017f000001" \
|
|
"fb3c1aaafb3c18fd00000f94000000360000000000000000942c0050061f04007f0000017f000001" \
|
|
"fb3c1cfcfb3c1a9b0000d3fc0000002a000000000000000000509434061b04007f0000017f000001" \
|
|
"fb3c1cfcfb3c1a9b00000a490000001e000000000000000094340050061f04007f0000017f000001" \
|
|
"fb3bb82cfb3ba48b000002960000000300000000000000000050942a061904007f0000017f000001" \
|
|
"fb3bb82cfb3ba48b00000068000000020000000000000000942a0050061104007f0000017f000001" \
|
|
"fb3c1900fb3c18fe0000004c0000000100000000000000000035b3c9110004007f0000017f000001" \
|
|
"fb3c1900fb3c18fe0000003c000000010000000000000000b3c9003511000400"
|
|
|
|
# This packet is special. We take PACKET_V9_TEMPLATE and re-order the templates and flows.
|
|
# The first line is the header, the smaller lines the templates and the long lines the flows (limited to 80 chars)
|
|
PACKET_V9_TEMPLATE_MIXED = ("0009000a000000035c9f55980000000100000000" # header
|
|
"040001447f0000017f000001fb3c1aaafb3c18fd000190100000004b00000000000000000050942c"
|
|
"061b04007f0000017f000001fb3c1aaafb3c18fd00000f94000000360000000000000000942c0050"
|
|
"061f04007f0000017f000001fb3c1cfcfb3c1a9b0000d3fc0000002a000000000000000000509434"
|
|
"061b04007f0000017f000001fb3c1cfcfb3c1a9b00000a490000001e000000000000000094340050"
|
|
"061f04007f0000017f000001fb3bb82cfb3ba48b000002960000000300000000000000000050942a"
|
|
"061904007f0000017f000001fb3bb82cfb3ba48b00000068000000020000000000000000942a0050"
|
|
"061104007f0000017f000001fb3c1900fb3c18fe0000004c0000000100000000000000000035b3c9"
|
|
"110004007f0000017f000001fb3c1900fb3c18fe0000003c000000010000000000000000b3c90035"
|
|
"11000400" # end of flow segments
|
|
"000000400400000e00080004000c000400150004001600040001000400020004" # template 1024
|
|
"000a0004000e000400070002000b00020004000100060001003c000100050001"
|
|
"000000400800000e001b0010001c001000150004001600040001000400020004" # template 2048
|
|
"000a0004000e000400070002000b00020004000100060001003c000100050001")
|
|
|
|
# Three packets without templates, each with 12 flows
|
|
PACKETS_V9 = [
|
|
"0009000c000000035c9f55980000000200000000040001e47f0000017f000001fb3c1a17fb3c19fd"
|
|
"000001480000000200000000000000000035ea82110004007f0000017f000001fb3c1a17fb3c19fd"
|
|
"0000007a000000020000000000000000ea820035110004007f0000017f000001fb3c1a17fb3c19fd"
|
|
"000000f80000000200000000000000000035c6e2110004007f0000017f000001fb3c1a17fb3c19fd"
|
|
"0000007a000000020000000000000000c6e20035110004007f0000017f000001fb3c1a9efb3c1a9c"
|
|
"0000004c0000000100000000000000000035adc1110004007f0000017f000001fb3c1a9efb3c1a9c"
|
|
"0000003c000000010000000000000000adc10035110004007f0000017f000001fb3c1b74fb3c1b72"
|
|
"0000004c0000000100000000000000000035d0b3110004007f0000017f000001fb3c1b74fb3c1b72"
|
|
"0000003c000000010000000000000000d0b30035110004007f0000017f000001fb3c2f59fb3c1b71"
|
|
"00001a350000000a000000000000000000509436061b04007f0000017f000001fb3c2f59fb3c1b71"
|
|
"0000038a0000000a000000000000000094360050061b04007f0000017f000001fb3c913bfb3c9138"
|
|
"0000004c0000000100000000000000000035e262110004007f0000017f000001fb3c913bfb3c9138"
|
|
"0000003c000000010000000000000000e262003511000400",
|
|
|
|
"0009000c000000035c9f55980000000300000000040001e47f0000017f000001fb3ca523fb3c913b"
|
|
"0000030700000005000000000000000000509438061b04007f0000017f000001fb3ca523fb3c913b"
|
|
"000002a200000005000000000000000094380050061b04007f0000017f000001fb3f7fe1fb3dbc97"
|
|
"0002d52800000097000000000000000001bb8730061b04007f0000017f000001fb3f7fe1fb3dbc97"
|
|
"0000146c000000520000000000000000873001bb061f04007f0000017f000001fb3d066ffb3d066c"
|
|
"0000004c0000000100000000000000000035e5bd110004007f0000017f000001fb3d066ffb3d066c"
|
|
"0000003c000000010000000000000000e5bd0035110004007f0000017f000001fb3d1a61fb3d066b"
|
|
"000003060000000500000000000000000050943a061b04007f0000017f000001fb3d1a61fb3d066b"
|
|
"000002a2000000050000000000000000943a0050061b04007f0000017f000001fb3fed00fb3f002c"
|
|
"0000344000000016000000000000000001bbae50061f04007f0000017f000001fb3fed00fb3f002c"
|
|
"00000a47000000120000000000000000ae5001bb061b04007f0000017f000001fb402f17fb402a75"
|
|
"0003524c000000a5000000000000000001bbc48c061b04007f0000017f000001fb402f17fb402a75"
|
|
"000020a60000007e0000000000000000c48c01bb061f0400",
|
|
|
|
"0009000c000000035c9f55980000000400000000040001e47f0000017f000001fb3d7ba2fb3d7ba0"
|
|
"0000004c0000000100000000000000000035a399110004007f0000017f000001fb3d7ba2fb3d7ba0"
|
|
"0000003c000000010000000000000000a3990035110004007f0000017f000001fb3d8f85fb3d7b9f"
|
|
"000003070000000500000000000000000050943c061b04007f0000017f000001fb3d8f85fb3d7b9f"
|
|
"000002a2000000050000000000000000943c0050061b04007f0000017f000001fb3d9165fb3d7f6d"
|
|
"0000c97b0000002a000000000000000001bbae48061b04007f0000017f000001fb3d9165fb3d7f6d"
|
|
"000007f40000001a0000000000000000ae4801bb061b04007f0000017f000001fb3dbc96fb3dbc7e"
|
|
"0000011e0000000200000000000000000035bd4f110004007f0000017f000001fb3dbc96fb3dbc7e"
|
|
"0000008e000000020000000000000000bd4f0035110004007f0000017f000001fb3ddbb3fb3c1a18"
|
|
"0000bfee0000002f00000000000000000050ae56061b04007f0000017f000001fb3ddbb3fb3c1a18"
|
|
"00000982000000270000000000000000ae560050061b04007f0000017f000001fb3ddbb3fb3c1a18"
|
|
"0000130e0000001200000000000000000050e820061b04007f0000017f000001fb3ddbb3fb3c1a18"
|
|
"0000059c000000140000000000000000e8200050061b0400"
|
|
]
|
|
|
|
# Example export for IPFIX (v10) with 4 templates, 1 option template and 8 data flow sets
|
|
PACKET_IPFIX_TEMPLATE = "000a05205e8465fd0000001300000000000200400400000e00080004000c00040016000400150004" \
|
|
"0001000400020004000a0004000e000400070002000b00020004000100060001003c000100050001" \
|
|
"000200340401000b00080004000c000400160004001500040001000400020004000a0004000e0004" \
|
|
"00200002003c000100050001000200400800000e001b0010001c0010001600040015000400010004" \
|
|
"00020004000a0004000e000400070002000b00020004000100060001003c00010005000100020034" \
|
|
"0801000b001b0010001c001000160004001500040001000400020004000a0004000e0004008b0002" \
|
|
"003c0001000500010003001e010000050001008f000400a000080131000401320004013000020100" \
|
|
"001a00000a5900000171352e67210000000100000000000104000054976500dfac110002ff7ed688" \
|
|
"ff7ed73a000015c70000000d000000000000000001bbe1a6061b0400ac110002976500dfff7ed688" \
|
|
"ff7ed73a0000074f000000130000000000000000e1a601bb061f04000401004cac110002ac110001" \
|
|
"ff7db9e0ff7dc1d0000000fc00000003000000000000000008000400ac110001ac110002ff7db9e0" \
|
|
"ff7dc1d0000000fc0000000300000000000000000000040008010220fde66f14e0f1960900000242" \
|
|
"ac110002ff0200000000000000000001ff110001ff7dfad6ff7e0e95000001b00000000600000000" \
|
|
"0000000087000600fde66f14e0f1960900000242ac110002fde66f14e0f196090000000000000001" \
|
|
"ff7e567fff7e664a0000020800000005000000000000000080000600fde66f14e0f1960900000000" \
|
|
"00000001fde66f14e0f1960900000242ac110002ff7e567fff7e664a000002080000000500000000" \
|
|
"0000000081000600fe800000000000000042aafffe73bbfafde66f14e0f1960900000242ac110002" \
|
|
"ff7e6aaaff7e6aaa0000004800000001000000000000000087000600fde66f14e0f1960900000242" \
|
|
"ac110002fe800000000000000042aafffe73bbfaff7e6aaaff7e6aaa000000400000000100000000" \
|
|
"0000000088000600fe800000000000000042acfffe110002fe800000000000000042aafffe73bbfa" \
|
|
"ff7e7eaaff7e7eaa0000004800000001000000000000000087000600fe800000000000000042aaff" \
|
|
"fe73bbfafe800000000000000042acfffe110002ff7e7eaaff7e7eaa000000400000000100000000" \
|
|
"0000000088000600fe800000000000000042aafffe73bbfafe800000000000000042acfffe110002" \
|
|
"ff7e92aaff7e92aa0000004800000001000000000000000087000600fe800000000000000042acff" \
|
|
"fe110002fe800000000000000042aafffe73bbfaff7e92aaff7e92aa000000400000000100000000" \
|
|
"000000008800060008000044fde66f14e0f1960900000242ac110002fd41b7143f86000000000000" \
|
|
"00000001ff7ec2a0ff7ec2a00000004a000000010000000000000000d20100351100060004000054" \
|
|
"ac1100027f000001ff7ed62eff7ed68700000036000000010000000000000000c496003511000400" \
|
|
"7f000001ac110002ff7ed62eff7ed687000000760000000100000000000000000035c49611000400" \
|
|
"08000044fde66f14e0f1960900000242ac110002fd41b7143f8600000000000000000001ff7ef359" \
|
|
"ff7ef3590000004a000000010000000000000000b1e700351100060004000054ac1100027f000001" \
|
|
"ff7f06e4ff7f06e800000036000000010000000000000000a8f90035110004007f000001ac110002" \
|
|
"ff7f06e4ff7f06e8000000a60000000100000000000000000035a8f911000400"
|
|
|
|
# Example export for IPFIX with two data sets
|
|
PACKET_IPFIX = "000a00d05e8465fd00000016000000000801007cfe800000000000000042acfffe110002fde66f14" \
|
|
"e0f196090000000000000001ff7f0755ff7f07550000004800000001000000000000000087000600" \
|
|
"fde66f14e0f196090000000000000001fe800000000000000042acfffe110002ff7f0755ff7f0755" \
|
|
"000000400000000100000000000000008800060008000044fde66f14e0f1960900000242ac110002" \
|
|
"2a044e42020000000000000000000223ff7f06e9ff7f22d500000140000000040000000000000000" \
|
|
"e54c01bb06020600"
|