From 337e80b670fc762ac0d321541fc51a313a4dc178 Mon Sep 17 00:00:00 2001 From: Jeremy Dubois Date: Tue, 21 Apr 2020 19:30:06 +0200 Subject: [PATCH 1/3] Writing of collectd role --- evolixisation.yml | 1 + roles/collectd/README.md | 13 +++ roles/collectd/defaults/main.yml | 32 ++++++ roles/collectd/files/dns_stats.sh | 3 + roles/collectd/files/ifq_drops.sh | 3 + roles/collectd/handlers/main.yml | 10 ++ roles/collectd/tasks/main.yml | 105 +++++++++++++++++++ roles/collectd/templates/collectd.conf.j2 | 122 ++++++++++++++++++++++ 8 files changed, 289 insertions(+) create mode 100644 roles/collectd/README.md create mode 100644 roles/collectd/defaults/main.yml create mode 100755 roles/collectd/files/dns_stats.sh create mode 100755 roles/collectd/files/ifq_drops.sh create mode 100644 roles/collectd/handlers/main.yml create mode 100644 roles/collectd/tasks/main.yml create mode 100644 roles/collectd/templates/collectd.conf.j2 diff --git a/evolixisation.yml b/evolixisation.yml index 03af200..950b46e 100644 --- a/evolixisation.yml +++ b/evolixisation.yml @@ -31,6 +31,7 @@ # - openvpn # - ospf # - bgp + # - { role: collectd, collectd_server: "127.0.0.1" } post_tasks: - include: "tasks/commit_etc_git.yml" diff --git a/roles/collectd/README.md b/roles/collectd/README.md new file mode 100644 index 0000000..e63b47b --- /dev/null +++ b/roles/collectd/README.md @@ -0,0 +1,13 @@ +# Collectd + +Installation and custom configuration of Collectd daemon. + +## Tasks + +Everything is in the `tasks/main.yml` file. + +## Available variables + +The full list of variables (with default values) can be found in `defaults/main.yml`. + +* `collectd_server` : server to which the data will be sent (default: 127.0.0.1). diff --git a/roles/collectd/defaults/main.yml b/roles/collectd/defaults/main.yml new file mode 100644 index 0000000..214234d --- /dev/null +++ b/roles/collectd/defaults/main.yml @@ -0,0 +1,32 @@ +--- + +# destination server + +collectd_server: "127.0.0.1" + +# execution interval + +collectd_interval: "300" + +# exec plugin + +collectd_plugin_exec: False # Set to true only if one of the exec plugins below is also set to true +collectd_plugin_exec_interval: "{{ collectd_interval }}" +collectd_plugin_exec_ifq_drops: False +collectd_plugin_exec_dns_stats: False # Based on unbound +collectd_plugin_exec_dns_stats_interval: "{{ collectd_interval }}" + +# others plugins + +collectd_plugin_cpu: True +collectd_plugin_df: True +collectd_plugin_disk: True +collectd_plugin_interface: True +collectd_plugin_load: True +collectd_plugin_memory: True +collectd_plugin_pf: True +collectd_plugin_processes: True +collectd_plugin_swap: True +collectd_plugin_tcpconns: True +collectd_plugin_uptime: True +collectd_plugin_users: True diff --git a/roles/collectd/files/dns_stats.sh b/roles/collectd/files/dns_stats.sh new file mode 100755 index 0000000..8a11d3d --- /dev/null +++ b/roles/collectd/files/dns_stats.sh @@ -0,0 +1,3 @@ +#!/bin/ksh + +echo "PUTVAL $(hostname)/dns_stats/count N:$(doas /bin/cat /var/log/daemon | grep "server stats" | grep -v "requestlist max" | awk '{print $13}' | tail -1)" diff --git a/roles/collectd/files/ifq_drops.sh b/roles/collectd/files/ifq_drops.sh new file mode 100755 index 0000000..25748f4 --- /dev/null +++ b/roles/collectd/files/ifq_drops.sh @@ -0,0 +1,3 @@ +#!/bin/ksh + +echo "PUTVAL $(hostname)/ifq_drops/count N:$(sysctl net.inet.ip.arpq.drops | awk -F= '{print $NF}')" diff --git a/roles/collectd/handlers/main.yml b/roles/collectd/handlers/main.yml new file mode 100644 index 0000000..00523b7 --- /dev/null +++ b/roles/collectd/handlers/main.yml @@ -0,0 +1,10 @@ +--- +- name: restart collectd + service: + name: collectd + state: restarted + +- name: reload unbound + service: + name: unbound + state: reloaded diff --git a/roles/collectd/tasks/main.yml b/roles/collectd/tasks/main.yml new file mode 100644 index 0000000..a678829 --- /dev/null +++ b/roles/collectd/tasks/main.yml @@ -0,0 +1,105 @@ +--- +- name: Install Collectd package + openbsd_pkg: + name: "collectd" + tags: + - collectd + +- name: Deploy Collectd configuration + template: + src: "collectd.conf.j2" + dest: "/etc/collectd.conf" + notify: restart collectd + tags: + - collectd + +- name: Enabling Collectd + service: + name: collectd + enabled: yes + tags: + - collectd + +- name: Create scripts directory for exec plugins + file: + path: /usr/local/share/collectd/scripts + state: directory + when: collectd_plugin_exec + tags: + - collectd + +- name: Copy ifq_drops.sh + copy: + src: ifq_drops.sh + dest: /usr/local/share/collectd/scripts/ifq_drops.sh + mode: 0755 + force: yes + when: collectd_plugin_exec_ifq_drops + tags: + - collectd + +- name: Remove ifq_drops.sh + file: + path: /usr/local/share/collectd/scripts/ifq_drops.sh + state: absent + when: not collectd_plugin_exec_ifq_drops + tags: + - collectd + +- name: Copy dns_stats.sh + copy: + src: dns_stats.sh + dest: /usr/local/share/collectd/scripts/dns_stats.sh + mode: 0755 + force: yes + when: collectd_plugin_exec_dns_stats + tags: + - collectd + +- name: Add stats DNS on unbound + lineinfile: + path: /var/unbound/etc/unbound.conf + regexp: 'statistics-interval' + line: ' statistics-interval: {{ collectd_plugin_exec_dns_stats_interval }}' + insertafter: 'hide-version:' + backup: yes + notify: reload unbound + when: collectd_plugin_exec_dns_stats + tags: + - collectd + +- name: Remove dns_stats.sh + file: + path: /usr/local/share/collectd/scripts/dns_stats.sh + state: absent + when: not collectd_plugin_exec_dns_stats + tags: + - collectd + +- name: Remove stats DNS on unbound + lineinfile: + path: /var/unbound/etc/unbound.conf + regexp: 'statistics-interval' + backup: yes + state: absent + notify: reload unbound + when: not collectd_plugin_exec_dns_stats + tags: + - collectd + +- name: Add doas configuration for dns_stats.sh execution + lineinfile: + path: /etc/doas.conf + line: 'permit nopass _collectd as root cmd /bin/cat' + when: collectd_plugin_exec_dns_stats + tags: + - collectd + +- name: Delete doas configuration for dns_stats.sh execution + lineinfile: + path: /etc/doas.conf + line: 'permit nopass _collectd as root cmd /bin/cat' + state: absent + when: not collectd_plugin_exec_dns_stats + tags: + - collectd diff --git a/roles/collectd/templates/collectd.conf.j2 b/roles/collectd/templates/collectd.conf.j2 new file mode 100644 index 0000000..6ce9e7d --- /dev/null +++ b/roles/collectd/templates/collectd.conf.j2 @@ -0,0 +1,122 @@ +Interval {{ collectd_interval }} +Timeout 2 + +LoadPlugin syslog + + LogLevel warning + + +{% if collectd_plugin_exec is sameas true %} + + Interval {{ collectd_plugin_exec_interval }} + + + +{% if collectd_plugin_exec_ifq_drops is sameas true %} + Exec "_collectd" "/usr/local/share/collectd/scripts/ifq_drops.sh" +{% endif %} +{% if collectd_plugin_exec_dns_stats is sameas true %} + Exec "_collectd" "/usr/local/share/collectd/scripts/dns_stats.sh" +{% endif %} + + +{% endif %} +{% if collectd_plugin_load is sameas true %} +LoadPlugin load +{% endif %} +{% if collectd_plugin_processes is sameas true %} +LoadPlugin processes +{% endif %} +{% if collectd_plugin_uptime is sameas true %} +LoadPlugin uptime +{% endif %} +{% if collectd_plugin_users is sameas true %} +LoadPlugin users +{% endif %} +{% if collectd_plugin_pf is sameas true %} +LoadPlugin pf +{% endif %} + +{% if collectd_plugin_df is sameas true %} +LoadPlugin df + + # expose host's mounts into container using -v /:/host:ro (location inside container does not matter much) + # ignore rootfs; else, the root file-system would appear twice, causing + # one of the updates to fail and spam the log + ## Seems to be fixed with collectd 5.5+ + ## FSType rootfs + # ignore the usual virtual / temporary file-systems + FSType sysfs + FSType proc + FSType devtmpfs + FSType devpts + FSType tmpfs + FSType fusectl + FSType cgroup + FSType overlay + FSType debugfs + FSType pstore + FSType securityfs + FSType hugetlbfs + FSType squashfs + FSType mqueue + IgnoreSelected true + + ReportByDevice false + ReportInodes true # Default false + ValuesAbsolute true + ValuesPercentage true + + +{% endif %} +{% if collectd_plugin_disk is sameas true %} +LoadPlugin disk + + #Disk "/^[hsv]d[a-z]/" + IgnoreSelected false + + +{% endif %} +{% if collectd_plugin_cpu is sameas true %} +LoadPlugin cpu + + ValuesPercentage true + + +{% endif %} +{% if collectd_plugin_memory is sameas true %} +LoadPlugin memory + + ValuesPercentage true + + +{% endif %} +{% if collectd_plugin_swap is sameas true %} +LoadPlugin swap + + ValuesPercentage true + + +{% endif %} +{% if collectd_plugin_interface is sameas true %} +LoadPlugin interface + + Interface "/^lo[0-9]*/" + Interface "/^veth.*/" + Interface "/^docker.*/" + IgnoreSelected true + ReportInactive false + + +{% endif %} +{% if collectd_plugin_tcpconns is sameas true %} +LoadPlugin tcpconns + + AllPortsSummary true + + +{% endif %} +LoadPlugin network + + Server "{{ collectd_server }}" "25826" + From 7cc374ea9e2245698880467e1f11f6c80cc18758 Mon Sep 17 00:00:00 2001 From: Jeremy Dubois Date: Mon, 12 Oct 2020 15:26:45 +0200 Subject: [PATCH 2/3] yamllint : indentation, trailing-spaces and truthy value --- roles/collectd/defaults/main.yml | 30 ++++++++++++------------ roles/collectd/tasks/main.yml | 39 ++++++++++++++++---------------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/roles/collectd/defaults/main.yml b/roles/collectd/defaults/main.yml index 214234d..8460a72 100644 --- a/roles/collectd/defaults/main.yml +++ b/roles/collectd/defaults/main.yml @@ -10,23 +10,23 @@ collectd_interval: "300" # exec plugin -collectd_plugin_exec: False # Set to true only if one of the exec plugins below is also set to true +collectd_plugin_exec: false # Set to true only if one of the exec plugins below is also set to true collectd_plugin_exec_interval: "{{ collectd_interval }}" -collectd_plugin_exec_ifq_drops: False -collectd_plugin_exec_dns_stats: False # Based on unbound +collectd_plugin_exec_ifq_drops: false +collectd_plugin_exec_dns_stats: false # Based on unbound collectd_plugin_exec_dns_stats_interval: "{{ collectd_interval }}" # others plugins -collectd_plugin_cpu: True -collectd_plugin_df: True -collectd_plugin_disk: True -collectd_plugin_interface: True -collectd_plugin_load: True -collectd_plugin_memory: True -collectd_plugin_pf: True -collectd_plugin_processes: True -collectd_plugin_swap: True -collectd_plugin_tcpconns: True -collectd_plugin_uptime: True -collectd_plugin_users: True +collectd_plugin_cpu: true +collectd_plugin_df: true +collectd_plugin_disk: true +collectd_plugin_interface: true +collectd_plugin_load: true +collectd_plugin_memory: true +collectd_plugin_pf: true +collectd_plugin_processes: true +collectd_plugin_swap: true +collectd_plugin_tcpconns: true +collectd_plugin_uptime: true +collectd_plugin_users: true diff --git a/roles/collectd/tasks/main.yml b/roles/collectd/tasks/main.yml index a678829..77a5988 100644 --- a/roles/collectd/tasks/main.yml +++ b/roles/collectd/tasks/main.yml @@ -3,7 +3,7 @@ openbsd_pkg: name: "collectd" tags: - - collectd + - collectd - name: Deploy Collectd configuration template: @@ -11,14 +11,14 @@ dest: "/etc/collectd.conf" notify: restart collectd tags: - - collectd + - collectd - name: Enabling Collectd service: name: collectd - enabled: yes + enabled: true tags: - - collectd + - collectd - name: Create scripts directory for exec plugins file: @@ -26,17 +26,17 @@ state: directory when: collectd_plugin_exec tags: - - collectd + - collectd - name: Copy ifq_drops.sh copy: src: ifq_drops.sh dest: /usr/local/share/collectd/scripts/ifq_drops.sh mode: 0755 - force: yes + force: true when: collectd_plugin_exec_ifq_drops tags: - - collectd + - collectd - name: Remove ifq_drops.sh file: @@ -44,29 +44,30 @@ state: absent when: not collectd_plugin_exec_ifq_drops tags: - - collectd + - collectd - name: Copy dns_stats.sh - copy: + copy: src: dns_stats.sh dest: /usr/local/share/collectd/scripts/dns_stats.sh mode: 0755 - force: yes + force: true when: collectd_plugin_exec_dns_stats tags: - - collectd + - collectd - name: Add stats DNS on unbound lineinfile: path: /var/unbound/etc/unbound.conf regexp: 'statistics-interval' - line: ' statistics-interval: {{ collectd_plugin_exec_dns_stats_interval }}' + line: + ' statistics-interval: {{ collectd_plugin_exec_dns_stats_interval }}' insertafter: 'hide-version:' - backup: yes + backup: true notify: reload unbound when: collectd_plugin_exec_dns_stats tags: - - collectd + - collectd - name: Remove dns_stats.sh file: @@ -74,18 +75,18 @@ state: absent when: not collectd_plugin_exec_dns_stats tags: - - collectd + - collectd - name: Remove stats DNS on unbound lineinfile: path: /var/unbound/etc/unbound.conf regexp: 'statistics-interval' - backup: yes + backup: true state: absent notify: reload unbound when: not collectd_plugin_exec_dns_stats tags: - - collectd + - collectd - name: Add doas configuration for dns_stats.sh execution lineinfile: @@ -93,7 +94,7 @@ line: 'permit nopass _collectd as root cmd /bin/cat' when: collectd_plugin_exec_dns_stats tags: - - collectd + - collectd - name: Delete doas configuration for dns_stats.sh execution lineinfile: @@ -102,4 +103,4 @@ state: absent when: not collectd_plugin_exec_dns_stats tags: - - collectd + - collectd From 11d3331958023766568cc3614b5a1ae0dcfb952b Mon Sep 17 00:00:00 2001 From: Jeremy Dubois Date: Mon, 12 Oct 2020 15:45:13 +0200 Subject: [PATCH 3/3] Collectd role : deletion of collectd_plugin_exec variable This variable had to be activated only if collectd_plugin_exec_ifq_drops or collectd_plugin_exec_dns_stats was also activated, for some configuration to be taken into account. I changed the role so that the configuration is automatically taken into account if one of these two variables is activated. --- roles/collectd/defaults/main.yml | 1 - roles/collectd/tasks/main.yml | 2 +- roles/collectd/templates/collectd.conf.j2 | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/roles/collectd/defaults/main.yml b/roles/collectd/defaults/main.yml index 8460a72..7974087 100644 --- a/roles/collectd/defaults/main.yml +++ b/roles/collectd/defaults/main.yml @@ -10,7 +10,6 @@ collectd_interval: "300" # exec plugin -collectd_plugin_exec: false # Set to true only if one of the exec plugins below is also set to true collectd_plugin_exec_interval: "{{ collectd_interval }}" collectd_plugin_exec_ifq_drops: false collectd_plugin_exec_dns_stats: false # Based on unbound diff --git a/roles/collectd/tasks/main.yml b/roles/collectd/tasks/main.yml index 77a5988..4ff066b 100644 --- a/roles/collectd/tasks/main.yml +++ b/roles/collectd/tasks/main.yml @@ -24,7 +24,7 @@ file: path: /usr/local/share/collectd/scripts state: directory - when: collectd_plugin_exec + when: collectd_plugin_exec_ifq_drops or collectd_plugin_exec_dns_stats tags: - collectd diff --git a/roles/collectd/templates/collectd.conf.j2 b/roles/collectd/templates/collectd.conf.j2 index 6ce9e7d..2cae0ac 100644 --- a/roles/collectd/templates/collectd.conf.j2 +++ b/roles/collectd/templates/collectd.conf.j2 @@ -6,7 +6,7 @@ LoadPlugin syslog LogLevel warning -{% if collectd_plugin_exec is sameas true %} +{% if (collectd_plugin_exec_ifq_drops is sameas true) or (collectd_plugin_exec_dns_stats is sameas true) %} Interval {{ collectd_plugin_exec_interval }}