From a9fe102f30d4bc250750deb5f80d21e5e67b635c Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Mon, 18 May 2020 12:03:34 +0200 Subject: [PATCH 01/88] packweb-apache: Don't turn on mod-evasive emails by default --- CHANGELOG.md | 2 ++ packweb-apache/templates/evolinux-evasive.conf.j2 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd045167..ab0d0ced 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ The **patch** part changes incrementally at each release. ### Changed +* packweb-apache: Don't turn on mod-evasive emails by default + ### Fixed ### Removed diff --git a/packweb-apache/templates/evolinux-evasive.conf.j2 b/packweb-apache/templates/evolinux-evasive.conf.j2 index fd73ad81..ce161b52 100644 --- a/packweb-apache/templates/evolinux-evasive.conf.j2 +++ b/packweb-apache/templates/evolinux-evasive.conf.j2 @@ -5,5 +5,5 @@ DOSSiteCount 30 DOSPageInterval 3 DOSSiteInterval 1 DOSBlockingPeriod 60 -DOSEmailNotify {{ general_alert_email }} +#DOSEmailNotify {{ general_alert_email }} From 8d9d068a3385647b25c45d479822b5fd490e02bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Thu, 28 May 2020 13:01:50 +0900 Subject: [PATCH 02/88] nqgios-nrpe: Add check_hpraid --- nagios-nrpe/files/plugins/check_hpraid | 266 +++++++++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 nagios-nrpe/files/plugins/check_hpraid diff --git a/nagios-nrpe/files/plugins/check_hpraid b/nagios-nrpe/files/plugins/check_hpraid new file mode 100644 index 00000000..3bd285cb --- /dev/null +++ b/nagios-nrpe/files/plugins/check_hpraid @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This check_hpraid is a fork from check_cciss v0.15 written by Simone Rosa. +# Upstream now at: +# https://gitea.evolix.org/evolix/ansible-roles/src/branch/stable/nagios-nrpe/files/plugins +# Source of the fork: +# https://exchange.nagios.org/directory/Plugins/Hardware/Storage-Systems/RAID-Controllers/check_cciss--2D-HP-and-Compaq-Smart-Array-Hardware-status/details +# +# Description: +# +# This plugin checks hardware status for Smart Array Controllers, +# using HPE Smart Storage Administrator. It should support Debian 9 and over. +# (Array, controller, cache, battery, etc...) +# +# NOTE: +# +# You need to install the proprietary tool HPE Smart Storage Administrator (ssacli) from: +# https://downloads.linux.hpe.com/SDR/repo/mcp +# Also NRPE need to launch ssacli as root. +# +# Please add this line to /etc/sudoers : +# -------------------------------------------------- +# nagios ALL=NOPASSWD: /usr/sbin/ssacli +# +# Examples: +# +# ./check_cciss +# ---------------- +# RAID OK +# +# ./check_cciss -v +# ------------------- +# RAID OK: Smart Array 6i in Slot 0 array A logicaldrive 1 (67.8 GB, RAID 1+0, OK) +# [Controller Status: OK Cache Status: OK Battery Status: OK] +# +# RAID CRITICAL - HP Smart Array Failed: Smart Array 6i in Slot 0 (Embedded) \ +# array A logicaldrive 1 (33.9 GB, RAID 1, Interim Recovery Mode) \ +# physicaldrive 1:0 (port 1:id 0 , Parallel SCSI, --- GB, Failed) +# +# RAID WARNING - HP Smart Array Rebuilding: Smart Array 6i in Slot 0 (Embedded) \ +# array A logicaldrive 1 (33.9 GB, RAID 1, Recovering, 26% complete) \ +# physicaldrive 1:0 (port 1:id 0 , Parallel SCSI, 36.4 GB, Rebuilding) +# +# ./check_cciss -v -p +# -------------------- +# RAID OK: Smart Array 6i in Slot 0 (Embedded) array A logicaldrive 1 (33.9 GB, RAID 1, OK) +# physicaldrive 2:0 (port 2:id 0 , Parallel SCSI, 36.4 GB, OK) +# physicaldrive 2:1 (port 2:id 1 , Parallel SCSI, 36.4 GB, OK) +# physicaldrive 1:5 (port 1:id 5 , Parallel SCSI, 72.8 GB, OK, spare) +# [Controller Status: OK Cache Status: OK Battery/Capacitor Status: OK] +# +# RAID CRITICAL - HP Smart Array Failed: Smart Array 6i in Slot 0 (Embedded) \ +# array A logicaldrive 1 (33.9 GB, RAID 1, Interim Recovery Mode) \ +# physicaldrive 1:0 (port 1:id 0 , Parallel SCSI, --- GB, Failed) \ +# physicaldrive 1:1 (port 1:id 1 , Parallel SCSI, 36.4 GB, OK) +# +# RAID WARNING - HP Smart Array Rebuilding: Smart Array 6i in Slot 0 (Embedded) \ +# array A logicaldrive 1 (33.9 GB, RAID 1, Recovering, 26% complete) \ +# physicaldrive 1:0 (port 1:id 0 , Parallel SCSI, 36.4 GB, Rebuilding) \ +# physicaldrive 1:1 (port 1:id 1 , Parallel SCSI, 36.4 GB, OK) +# +# ./check_cciss -v -b +# ---------------- +# +# RAID OK: Smart Array 6i in Slot 0 (Embedded) array A logicaldrive 1 (33.9 GB, RAID 1, OK) [Controller Status: OK] +# +# [insted of] +# RAID CRITICAL - HP Smart Array Failed: Smart Array 6i in Slot 0 (Embedded) \ +# Controller Status: OK Cache Status: Temporarily Disabled \ +# Battery/Capacitor Status: Failed (Replace Batteries/Capacitors) + +PROGNAME=$(basename $0) +NAGIOS_PLUGINS="/usr/lib/nagios/plugins/" +REVISION=$(echo '0.16-evolix') +DEBUG="0" +VERBOSE="0" +ssacli=$(command -v ssacli) +PHYSICAL_DRIVE=0 + +. ${NAGIOS_PLUGINS}/utils.sh + +print_usage() { + echo "" + echo "Usage: $PROGNAME [-v] [-p] [-e ] [-E ] [-b] [-s] [-d]" + echo "Usage: $PROGNAME [-h]" + echo "Usage: $PROGNAME [-V]" + echo "" + echo " -v = show status and informations about RAID" + echo " -p = show detail for physical drives" + echo " -e = exclude slot number" + echo " -b = exclude battery/capacitor/cache status check" + echo " -d = use for debug (command line mode)" + echo " -h = help information" + echo " -V = version information" + echo "" + echo " =============" +} + +print_help() { + print_revision $PROGNAME $REVISION + echo "" + print_usage + echo "" + echo "This plugin checks hardware status for Smart Array Controllers," + echo "using HPE Smart Storage Administrator." + echo "" + support + exit 0 +} + +while getopts "N:cvpbsde:Vh" options +do + case $options in + N) ;; + c) ;; + v) VERBOSE=1;; + p) PHYSICAL_DRIVE=1;; + d) DEBUG=1;; + e) EXCLUDE_SLOT=1 + excludeslot="$OPTARG";; + b) EXCLUDE_BATTERY=1;; + V) print_revision $PROGNAME $REVISION + exit 0;; + h) print_help + exit 0;; + \?) print_usage + exit 0;; + *) print_usage + exit 0;; + esac +done + +# Use smartpqi driver +# https://manpages.debian.org/buster/manpages/smartpqi.4.en.html +if [ -d /sys/bus/pci/drivers/smartpqi ]; then + DRIVER="/sys/bus/pci/drivers/smartpqi" + driverPresent='YES.' +else + driverPresent='No!' +fi +if [ "$DEBUG" = "1" ]; then + echo "### Check if \"HP Smart Array\" ($DRIVER) is present >>>\n"${driverPresent}"\n" +fi +if [[ "$driverPresent" == "No!" ]]; then + echo "RAID UNKNOWN - HP Smart Array not found" + exit $STATE_UNKNOWN +fi + +# Check if "HP Array Utility CLI" is present +if [ "$DEBUG" = "1" ]; then + echo "### Check if \"ssacli\" is present >>>\n" +fi +if [ ! -x $ssacli ]; then + if [ -x $ssacli ]; then + if [ "$DEBUG" = "1" ]; then + echo "### \"ssacli\" is present >>>\n" + fi + else + echo "ERROR: ssacli tools should be installed and with right sudoers/permissions (see the notes above)" + exit $STATE_UNKNOWN + fi +fi + +# Check if "HP Controller" work correctly +check=$(sudo -u root $ssacli controller all show status 2>&1) +status=$? +if [ "$DEBUG" = "1" ]; then + echo "### Check if \"HP Controller\" work correctly >>>\n"${check}"\n" +fi +if test ${status} -ne 0; then + echo "RAID UNKNOWN - $ssacli did not execute properly : "${check} + exit $STATE_UNKNOWN +fi + +# Get "Slot" & exclude slot needed +EXCLUDE_SLOT=${EXCLUDE_SLOT:-0} +if [ "$EXCLUDE_SLOT" = "1" ]; then + slots=$(echo ${check} | egrep -o "Slot \w" | awk '{print $NF}' | grep -v "$excludeslot") +else + slots=$(echo ${check} | egrep -o "Slot \w" | awk '{print $NF}') +fi +if [ "$DEBUG" = "1" ]; then + echo "### Get \"Slot\" & exclude slot not needed >>>\n"${slots}"\n" +fi +for slot in $slots; do + # Get "logicaldrive" for slot + check2b=$(sudo -u root $ssacli controller slot=$slot logicaldrive all show 2>&1) + status=$? + if test ${status} -ne 0; then + echo "RAID UNKNOWN - $ssacli did not execute properly : "${check2b} + exit $STATE_UNKNOWN + fi + check2=${check2:-} + check2="$check2$check2b" + if [ "$DEBUG" = "1" ]; then + echo "### Get \"logicaldrive\" for slot >>>\n"${check2b}"\n" + fi + + # Get "physicaldrive" for slot + if [ "$PHYSICAL_DRIVE" = "1" -o "$DEBUG" = "1" ]; then + check2b=$(sudo -u root $ssacli controller slot=$slot physicaldrive all show | sed -e 's/\?/\-/g' 2>&1 | grep "physicaldrive") + else + check2b=$(sudo -u root $ssacli controller slot=$slot physicaldrive all show | sed -e 's/\?/\-/g' 2>&1 | grep "physicaldrive" | (grep "\(Failure\|Failed\|Rebuilding\)" || true)) + fi + status=$? + if [ "$PHYSICAL_DRIVE" = "1" -o "$DEBUG" = "1" ]; then + if test ${status} -ne 0; then + echo "RAID UNKNOWN - $ssacli did not execute properly : "${check2b} + exit $STATE_UNKNOWN + fi + fi + check2="$check2$check2b" + if [ "$DEBUG" = "1" ]; then + echo "### Get \"physicaldrive\" for slot >>>\n"${check2b}"\n" + fi +done + +# Check STATUS +if [ "$DEBUG" = "1" ]; then + echo "### Check STATUS >>>" +fi + +# Omit battery/capacitor/cache status check if requested +EXCLUDE_BATTERY=${EXCLUDE_BATTERY:-0} +if [ "$EXCLUDE_BATTERY" = "1" ]; then + check=$(echo "$check" | grep -v 'Battery/Capacitor Status: Failed (Replace Batteries/Capacitors)') + check=$(echo "$check" | grep -v 'Cache Status: Temporarily Disabled') +fi + +if echo ${check} | egrep Failed >/dev/null; then + echo "RAID CRITICAL - HP Smart Array Failed: "${check} | egrep Failed + exit $STATE_CRITICAL +elif echo ${check} | egrep Disabled >/dev/null; then + echo "RAID CRITICAL - HP Smart Array Problem: "${check} | egrep Disabled + exit $STATE_CRITICAL +elif echo ${check2} | egrep Failed >/dev/null; then + echo "RAID CRITICAL - HP Smart Array Failed: "${check2} | egrep Failed + exit $STATE_CRITICAL +elif echo ${check2} | egrep Failure >/dev/null; then + echo "RAID WARNING - Component Failure: "${check2} | egrep Failure + exit $STATE_WARNING +elif echo ${check2} | egrep Rebuild >/dev/null; then + echo "RAID WARNING - HP Smart Array Rebuilding: "${check2} | egrep Rebuild + exit $STATE_WARNING +elif echo ${check2} | egrep Recover >/dev/null; then + echo "RAID WARNING - HP Smart Array Recovering: "${check2} | egrep Recover + exit $STATE_WARNING +elif echo ${check} | egrep "Cache Status: Temporarily Disabled" >/dev/null; then + echo "RAID WARNING - HP Smart Array Cache Disabled: "${check} + exit $STATE_WARNING +elif echo ${check} | egrep FIRMWARE >/dev/null; then + echo "RAID WARNING - "${check} + exit $STATE_WARNING +else + if [ "$DEBUG" = "1" -o "$VERBOSE" = "1" ]; then + check3=$(echo "${check}" | egrep Status) + check3=$(echo ${check3}) + echo "RAID OK: "${check2}" ["${check3}"]" + else + echo "RAID OK" + fi + exit $STATE_OK +fi + +exit $STATE_UNKNOWN From d8a3bd0c016ce5f5a0d811736975e767046780c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Thu, 28 May 2020 15:33:00 +0900 Subject: [PATCH 03/88] nagios-nrpe: Adding licence GPLv2 to check_hpraid Also describe what has been removed/changed from the original source. --- nagios-nrpe/files/plugins/check_hpraid | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nagios-nrpe/files/plugins/check_hpraid b/nagios-nrpe/files/plugins/check_hpraid index 3bd285cb..b5514a87 100644 --- a/nagios-nrpe/files/plugins/check_hpraid +++ b/nagios-nrpe/files/plugins/check_hpraid @@ -2,11 +2,13 @@ set -euo pipefail # This check_hpraid is a fork from check_cciss v0.15 written by Simone Rosa. +# Usage of old tools and drivers were removed to use only the smartpqi driver and the ssacli tool from HP. # Upstream now at: # https://gitea.evolix.org/evolix/ansible-roles/src/branch/stable/nagios-nrpe/files/plugins # Source of the fork: # https://exchange.nagios.org/directory/Plugins/Hardware/Storage-Systems/RAID-Controllers/check_cciss--2D-HP-and-Compaq-Smart-Array-Hardware-status/details -# +# +# Licence: GPLv2 # Description: # # This plugin checks hardware status for Smart Array Controllers, From 5bf7ba5c84cef6dee767b8daca16eee2291f49f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Thu, 28 May 2020 16:35:11 +0900 Subject: [PATCH 04/88] nagios-nrpe: shellchecked check_hp Also refactored the checking part and outputing. --- nagios-nrpe/files/plugins/check_hpraid | 125 +++++++++++++------------ 1 file changed, 65 insertions(+), 60 deletions(-) diff --git a/nagios-nrpe/files/plugins/check_hpraid b/nagios-nrpe/files/plugins/check_hpraid index b5514a87..1bac12e6 100644 --- a/nagios-nrpe/files/plugins/check_hpraid +++ b/nagios-nrpe/files/plugins/check_hpraid @@ -1,9 +1,13 @@ #!/usr/bin/env bash +# shellcheck disable=SC2028 set -euo pipefail # This check_hpraid is a fork from check_cciss v0.15 written by Simone Rosa. +# Fork written by Evolix and for Evolix usage (Debian only). # Usage of old tools and drivers were removed to use only the smartpqi driver and the ssacli tool from HP. -# Upstream now at: +# Tools not used on Debian were also removed. +# Linting tool shellcheck was used to use a better bash coding style. +# Upstream at: # https://gitea.evolix.org/evolix/ansible-roles/src/branch/stable/nagios-nrpe/files/plugins # Source of the fork: # https://exchange.nagios.org/directory/Plugins/Hardware/Storage-Systems/RAID-Controllers/check_cciss--2D-HP-and-Compaq-Smart-Array-Hardware-status/details @@ -72,14 +76,15 @@ set -euo pipefail # Controller Status: OK Cache Status: Temporarily Disabled \ # Battery/Capacitor Status: Failed (Replace Batteries/Capacitors) -PROGNAME=$(basename $0) +PROGNAME=$(basename "$0") NAGIOS_PLUGINS="/usr/lib/nagios/plugins/" -REVISION=$(echo '0.16-evolix') +REVISION="0.16-evolix" DEBUG="0" VERBOSE="0" ssacli=$(command -v ssacli) PHYSICAL_DRIVE=0 +# shellcheck source=/dev/null . ${NAGIOS_PLUGINS}/utils.sh print_usage() { @@ -100,7 +105,7 @@ print_usage() { } print_help() { - print_revision $PROGNAME $REVISION + print_revision "$PROGNAME" "$REVISION" echo "" print_usage echo "" @@ -122,7 +127,7 @@ do e) EXCLUDE_SLOT=1 excludeslot="$OPTARG";; b) EXCLUDE_BATTERY=1;; - V) print_revision $PROGNAME $REVISION + V) print_revision "$PROGNAME" "$REVISION" exit 0;; h) print_help exit 0;; @@ -142,79 +147,80 @@ else driverPresent='No!' fi if [ "$DEBUG" = "1" ]; then - echo "### Check if \"HP Smart Array\" ($DRIVER) is present >>>\n"${driverPresent}"\n" + echo "### Check if \"HP Smart Array\" ($DRIVER) is present >>>\n${driverPresent}\n" fi if [[ "$driverPresent" == "No!" ]]; then echo "RAID UNKNOWN - HP Smart Array not found" - exit $STATE_UNKNOWN + exit "$STATE_UNKNOWN" fi # Check if "HP Array Utility CLI" is present if [ "$DEBUG" = "1" ]; then echo "### Check if \"ssacli\" is present >>>\n" fi -if [ ! -x $ssacli ]; then - if [ -x $ssacli ]; then +if [ ! -x "$ssacli" ]; then + if [ -x "$ssacli" ]; then if [ "$DEBUG" = "1" ]; then echo "### \"ssacli\" is present >>>\n" fi else echo "ERROR: ssacli tools should be installed and with right sudoers/permissions (see the notes above)" - exit $STATE_UNKNOWN + exit "$STATE_UNKNOWN" fi fi # Check if "HP Controller" work correctly -check=$(sudo -u root $ssacli controller all show status 2>&1) +check=$(sudo -u root "$ssacli" controller all show status 2>&1) status=$? if [ "$DEBUG" = "1" ]; then - echo "### Check if \"HP Controller\" work correctly >>>\n"${check}"\n" + echo "### Check if \"HP Controller\" work correctly >>>\n""${check}""\n" fi if test ${status} -ne 0; then - echo "RAID UNKNOWN - $ssacli did not execute properly : "${check} - exit $STATE_UNKNOWN + echo "RAID UNKNOWN - $ssacli did not execute properly : ""${check}" + exit "$STATE_UNKNOWN" fi # Get "Slot" & exclude slot needed EXCLUDE_SLOT=${EXCLUDE_SLOT:-0} if [ "$EXCLUDE_SLOT" = "1" ]; then - slots=$(echo ${check} | egrep -o "Slot \w" | awk '{print $NF}' | grep -v "$excludeslot") + slots=$(grep -E -o "Slot \w" <<< "$check" | awk '{print $NF}' | grep -v "$excludeslot") else - slots=$(echo ${check} | egrep -o "Slot \w" | awk '{print $NF}') + slots=$(grep -E -o "Slot \w" <<< "$check" | awk '{print $NF}') fi if [ "$DEBUG" = "1" ]; then - echo "### Get \"Slot\" & exclude slot not needed >>>\n"${slots}"\n" + echo "### Get \"Slot\" & exclude slot not needed >>>\n""${slots}""\n" fi + for slot in $slots; do # Get "logicaldrive" for slot - check2b=$(sudo -u root $ssacli controller slot=$slot logicaldrive all show 2>&1) + check2b=$(sudo -u root "$ssacli" controller slot="$slot" logicaldrive all show 2>&1) status=$? if test ${status} -ne 0; then - echo "RAID UNKNOWN - $ssacli did not execute properly : "${check2b} - exit $STATE_UNKNOWN + echo "RAID UNKNOWN - $ssacli did not execute properly : ""${check2b}" + exit "$STATE_UNKNOWN" fi check2=${check2:-} check2="$check2$check2b" if [ "$DEBUG" = "1" ]; then - echo "### Get \"logicaldrive\" for slot >>>\n"${check2b}"\n" + echo "### Get \"logicaldrive\" for slot >>>\n""${check2b}""\n" fi # Get "physicaldrive" for slot - if [ "$PHYSICAL_DRIVE" = "1" -o "$DEBUG" = "1" ]; then - check2b=$(sudo -u root $ssacli controller slot=$slot physicaldrive all show | sed -e 's/\?/\-/g' 2>&1 | grep "physicaldrive") + if [ "$PHYSICAL_DRIVE" = "1" ] || [ "$DEBUG" = "1" ]; then + check2b=$(sudo -u root "$ssacli" controller slot="$slot" physicaldrive all show | sed -e 's/\?/\-/g' 2>&1 | grep "physicaldrive") else - check2b=$(sudo -u root $ssacli controller slot=$slot physicaldrive all show | sed -e 's/\?/\-/g' 2>&1 | grep "physicaldrive" | (grep "\(Failure\|Failed\|Rebuilding\)" || true)) + check2b=$(sudo -u root "$ssacli" controller slot="$slot" physicaldrive all show | sed -e 's/\?/\-/g' 2>&1 | grep "physicaldrive" | (grep "\(Failure\|Failed\|Rebuilding\)" || true)) fi status=$? - if [ "$PHYSICAL_DRIVE" = "1" -o "$DEBUG" = "1" ]; then + if [ "$PHYSICAL_DRIVE" = "1" ] || [ "$DEBUG" = "1" ]; then if test ${status} -ne 0; then - echo "RAID UNKNOWN - $ssacli did not execute properly : "${check2b} - exit $STATE_UNKNOWN + echo "RAID UNKNOWN - $ssacli did not execute properly : ""${check2b}" + exit "$STATE_UNKNOWN" fi fi check2="$check2$check2b" if [ "$DEBUG" = "1" ]; then - echo "### Get \"physicaldrive\" for slot >>>\n"${check2b}"\n" + echo "### Get \"physicaldrive\" for slot >>>\n""${check2b}""\n" fi done @@ -226,43 +232,42 @@ fi # Omit battery/capacitor/cache status check if requested EXCLUDE_BATTERY=${EXCLUDE_BATTERY:-0} if [ "$EXCLUDE_BATTERY" = "1" ]; then - check=$(echo "$check" | grep -v 'Battery/Capacitor Status: Failed (Replace Batteries/Capacitors)') - check=$(echo "$check" | grep -v 'Cache Status: Temporarily Disabled') + check=$(grep -v 'Battery/Capacitor Status: Failed (Replace Batteries/Capacitors)' "$check") + check=$(grep -v 'Cache Status: Temporarily Disabled' "$check") fi -if echo ${check} | egrep Failed >/dev/null; then - echo "RAID CRITICAL - HP Smart Array Failed: "${check} | egrep Failed - exit $STATE_CRITICAL -elif echo ${check} | egrep Disabled >/dev/null; then - echo "RAID CRITICAL - HP Smart Array Problem: "${check} | egrep Disabled - exit $STATE_CRITICAL -elif echo ${check2} | egrep Failed >/dev/null; then - echo "RAID CRITICAL - HP Smart Array Failed: "${check2} | egrep Failed - exit $STATE_CRITICAL -elif echo ${check2} | egrep Failure >/dev/null; then - echo "RAID WARNING - Component Failure: "${check2} | egrep Failure - exit $STATE_WARNING -elif echo ${check2} | egrep Rebuild >/dev/null; then - echo "RAID WARNING - HP Smart Array Rebuilding: "${check2} | egrep Rebuild - exit $STATE_WARNING -elif echo ${check2} | egrep Recover >/dev/null; then - echo "RAID WARNING - HP Smart Array Recovering: "${check2} | egrep Recover - exit $STATE_WARNING -elif echo ${check} | egrep "Cache Status: Temporarily Disabled" >/dev/null; then - echo "RAID WARNING - HP Smart Array Cache Disabled: "${check} - exit $STATE_WARNING -elif echo ${check} | egrep FIRMWARE >/dev/null; then - echo "RAID WARNING - "${check} - exit $STATE_WARNING +if grep -qiE Failed <<< "$check"; then + echo "RAID CRITICAL - HP Smart Array Failed: ${check}" + exit "$STATE_CRITICAL" +elif grep -qiE Disabled <<< "$check"; then + echo "RAID CRITICAL - HP Smart Array Problem: ${check}" + exit "$STATE_CRITICAL" +elif grep -qiE Failed <<< "$check2"; then + echo "RAID CRITICAL - HP Smart Array Failed: ${check2}" + exit "$STATE_CRITICAL" +elif grep -qiE Failure <<< "$check2"; then + echo "RAID WARNING - Component Failure: ${check2}" + exit "$STATE_WARNING" +elif grep -qiE Rebuild <<< "$check2"; then + echo "RAID WARNING - HP Smart Array Rebuilding: ${check2}" + exit "$STATE_WARNING" +elif grep -qiE Recover <<< "$check2"; then + echo "RAID WARNING - HP Smart Array Recovering: ${check2}" + exit "$STATE_WARNING" +elif grep -qiE "Cache Status: Temporarily Disabled" <<< "$check"; then + echo "RAID WARNING - HP Smart Array Cache Disabled: ${check}" + exit "$STATE_WARNING" +elif grep -qiE FIRMWARE <<< "$check"; then + echo "RAID WARNING - ${check}" + exit "$STATE_WARNING" else - if [ "$DEBUG" = "1" -o "$VERBOSE" = "1" ]; then - check3=$(echo "${check}" | egrep Status) - check3=$(echo ${check3}) - echo "RAID OK: "${check2}" ["${check3}"]" + if [ "$DEBUG" = "1" ] || [ "$VERBOSE" = "1" ]; then + check3=$(grep -E Status <<< "$check") + echo "RAID OK: ${check2} [${check3}]" else echo "RAID OK" fi - exit $STATE_OK + exit "$STATE_OK" fi -exit $STATE_UNKNOWN +exit "$STATE_UNKNOWN" From d0004ebaccb5db16b0dfdcfe50772e5691a6db38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Fri, 29 May 2020 09:43:15 +0900 Subject: [PATCH 05/88] nagios-nrpe: Added hpsa support to check_hpraid Also handle empty slots. --- nagios-nrpe/files/plugins/check_hpraid | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/nagios-nrpe/files/plugins/check_hpraid b/nagios-nrpe/files/plugins/check_hpraid index 1bac12e6..2dc567b4 100644 --- a/nagios-nrpe/files/plugins/check_hpraid +++ b/nagios-nrpe/files/plugins/check_hpraid @@ -4,7 +4,7 @@ set -euo pipefail # This check_hpraid is a fork from check_cciss v0.15 written by Simone Rosa. # Fork written by Evolix and for Evolix usage (Debian only). -# Usage of old tools and drivers were removed to use only the smartpqi driver and the ssacli tool from HP. +# Usage of old tools and drivers were removed to use only the smartpqi or hpsa drivers and the ssacli tool from HP. # Tools not used on Debian were also removed. # Linting tool shellcheck was used to use a better bash coding style. # Upstream at: @@ -77,7 +77,7 @@ set -euo pipefail # Battery/Capacitor Status: Failed (Replace Batteries/Capacitors) PROGNAME=$(basename "$0") -NAGIOS_PLUGINS="/usr/lib/nagios/plugins/" +NAGIOS_PLUGINS="/usr/lib/nagios/plugins" REVISION="0.16-evolix" DEBUG="0" VERBOSE="0" @@ -138,16 +138,15 @@ do esac done -# Use smartpqi driver +# Check if smartpqi or hpsa driver is loaded # https://manpages.debian.org/buster/manpages/smartpqi.4.en.html -if [ -d /sys/bus/pci/drivers/smartpqi ]; then - DRIVER="/sys/bus/pci/drivers/smartpqi" +if [ -d /sys/bus/pci/drivers/smartpqi ] || [ -d /sys/bus/pci/drivers/hpsa ]; then driverPresent='YES.' else driverPresent='No!' fi if [ "$DEBUG" = "1" ]; then - echo "### Check if \"HP Smart Array\" ($DRIVER) is present >>>\n${driverPresent}\n" + echo "### Check if \"HP Smart Array\" driver is present >>>\n${driverPresent}\n" fi if [[ "$driverPresent" == "No!" ]]; then echo "RAID UNKNOWN - HP Smart Array not found" @@ -193,12 +192,18 @@ fi for slot in $slots; do # Get "logicaldrive" for slot + set +e check2b=$(sudo -u root "$ssacli" controller slot="$slot" logicaldrive all show 2>&1) status=$? if test ${status} -ne 0; then + # Skip empty slots + if grep -q "The specified device does not have any logical drives." <<< "$check2b"; then + break + fi echo "RAID UNKNOWN - $ssacli did not execute properly : ""${check2b}" exit "$STATE_UNKNOWN" fi + set -e check2=${check2:-} check2="$check2$check2b" if [ "$DEBUG" = "1" ]; then From 6a264a31797ea57f47c6dfef83c146e67f8e480b Mon Sep 17 00:00:00 2001 From: Patrick Marchand Date: Thu, 17 Oct 2019 13:35:11 -0400 Subject: [PATCH 06/88] Added option to prepare mysql servers for replication --- mysql/README.md | 11 ++- mysql/defaults/main.yml | 10 ++- mysql/files/dbadmin.sh | 101 +++++++++++++++++++++++ mysql/files/xinetd/mysqlchk | 13 +++ mysql/files/xinetd/mysqlchk.sh | 54 ++++++++++++ mysql/handlers/main.yml | 5 ++ mysql/tasks/main.yml | 3 + mysql/tasks/replication.yml | 53 ++++++++++++ mysql/templates/evolinux-custom.cnf.j2 | 7 +- mysql/templates/replication.cnf.j2 | 7 ++ mysql/templates/replication_check.cfg.j2 | 3 + 11 files changed, 255 insertions(+), 12 deletions(-) create mode 100644 mysql/files/dbadmin.sh create mode 100644 mysql/files/xinetd/mysqlchk create mode 100644 mysql/files/xinetd/mysqlchk.sh create mode 100644 mysql/tasks/replication.yml create mode 100644 mysql/templates/replication.cnf.j2 create mode 100644 mysql/templates/replication_check.cfg.j2 diff --git a/mysql/README.md b/mysql/README.md index ff4fcbe9..bb87921e 100644 --- a/mysql/README.md +++ b/mysql/README.md @@ -15,11 +15,13 @@ Tasks are extracted in several files, included in `tasks/main.yml` : * `munin.yml` : Munin plugins ; * `log2mail.yml` : log2mail patterns ; * `utils.yml` : useful tools. +* `replication.yml`: install and configure prerequisites for mysql replication, do not forget to set `mysql_bind_address`, `mysql_server_id` and `mysql_log_bin` ## Available variables * `mysql_variant` : install Oracle's MySQL or MariaDB (default: `oracle`) [Debian 8 only]; * `mysql_replace_root_with_mysqladmin`: switch from `root` to `mysqladmin` user or not ; +* `mysql_replication`: setup all prerequisites for replication. * `mysql_thread_cache_size`: number of threads for the cache ; * `mysql_innodb_buffer_pool_size`: amount of RAM dedicated to InnoDB ; * `mysql_bind_address` : (default: `Null`, default evolinux config is then used) ; @@ -30,8 +32,7 @@ Tasks are extracted in several files, included in `tasks/main.yml` : * `mysql_max_heap_table_size`: (default: `Null`, default evolinux config is then used) ; * `mysql_query_cache_limit`: (default: `Null`, default evolinux config is then used) ; * `mysql_query_cache_size`: (default: `Null`, default evolinux config is then used) ; -* `mysql_log_bin`: (default: `Null`, activates binlogs if used) ; -* `mysql_server_id`: (default: `Null`, MySQL version default is then used) ; +* `mysql_server_id`: (default: `Null`, only used with `mysql_replication`, default mysql server id will be used otherwise) ; * `mysql_custom_datadir`: custom datadir. * `mysql_custom_tmpdir`: custom tmpdir. * `general_alert_email`: email address to send various alert messages (default: `root@localhost`). @@ -41,5 +42,9 @@ Tasks are extracted in several files, included in `tasks/main.yml` : * `mysql_force_new_nrpe_password` : change the password for NRPE even if it exists already (default: `False`). * `mysql_install_libclient`: install mysql client libraries (default: `False`). * `mysql_restart_if_needed` : should the restart handler be executed (default: `True`) +* `mysql_log_bin`: (default: `Null`, activates binlogs if used with `mysql_replication`) ; +* `mysql_repl_password`: Password hash for replication user, only creates a user if set. +## Notes +Changing the _datadir_ location can be done multiple times, as long as it is not restored to the default initial location, (because a symlink is created and can't be switched back, yet). -NB : changing the _datadir_ location can be done multiple times, as long as it is not restored to the default initial location, (because a symlink is created and can't be switched back, yet). +When using replication, note that the connections from the client server on the haproxy 8306 and mysql 3306 ports need to be open and the sql servers need to communicate on port 3306. diff --git a/mysql/defaults/main.yml b/mysql/defaults/main.yml index 633619cf..f364de18 100644 --- a/mysql/defaults/main.yml +++ b/mysql/defaults/main.yml @@ -21,7 +21,6 @@ mysql_innodb_buffer_pool_size: '{{ (ansible_memtotal_mb * 0.3) | int }}M' # If these variables are changed to non-Null values, # they will be added in the zzz-evolinux-custom.cnf file. # Otherwise, the value from de the z-evolinux-defaults.cnf file will preveil. -mysql_bind_address: Null mysql_max_connections: Null mysql_max_connect_errors: Null mysql_table_cache: Null @@ -29,8 +28,6 @@ mysql_tmp_table_size: Null mysql_max_heap_table_size: Null mysql_query_cache_limit: Null mysql_query_cache_size: Null -mysql_log_bin: Null -mysql_server_id: Null mysql_cron_optimize: True mysql_cron_optimize_frequency: weekly @@ -44,3 +41,10 @@ mysql_evolinux_defaults_file: z-evolinux-defaults.cnf mysql_evolinux_custom_file: zzz-evolinux-custom.cnf mysql_restart_if_needed: True + +# replication variables: +mysql_replication: false +mysql_log_bin: null +mysql_server_id: null +mysql_bind_address: null +mysql_repl_password: '' \ No newline at end of file diff --git a/mysql/files/dbadmin.sh b/mysql/files/dbadmin.sh new file mode 100644 index 00000000..f5e61ea8 --- /dev/null +++ b/mysql/files/dbadmin.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# +# Manage MySQL accounts and databases. +# +# Note: in the following code: +# - account means user@host +# - user is the user part of account +# + +MYSQL_OPTS="--raw --skip-column-names --skip-line-numbers" + +usage() { + cat <&2 +Usage: $0 [] + +Available commands are: + + list [] + List all accounts and their databases, separated by semi-colon. If user + is specified, list databases for this user only. + + passwd + Change password for specified user. + +EOT +} + +error() { + printf >&2 "Error: $@\n" +} + +get_host() { + user="$1" + host=$(mysql $MYSQL_OPTS --execute "SELECT host FROM mysql.user WHERE user='$user'") + if [ $(echo "$host" |wc -l) -gt 1 ]; then + # TODO: Not perfect! + echo "$host" |grep '%' + else + echo $host + fi +} + +get_dbs() { + account="$1" + echo "$(mysql $MYSQL_OPTS --execute "SHOW GRANTS FOR $account" |perl -ne 'print "$1 " if (/^GRANT (?!USAGE).* ON `(.*)`/)')" +} + +get_accounts() { + echo "$(mysql $MYSQL_OPTS --execute "SELECT user,host FROM mysql.user;" |perl -ne 'print "$1\@$2\n" if (/^([^\s]+)\s+([^\s]+)$/)'|sed "s/^/'/; s/@/'@'/; s/$/'/;")" +} + +list() { + if [ $# -gt 0 ]; then + user="$1" + host=$(get_host $user) + account="'$user'@'$host'" + echo $account:$(get_dbs "$account") + else + for account in $(get_accounts); do + echo $account:$(get_dbs "$account") + done + fi +} + +passwd() { + if [ $# -ne 2 ]; then + usage + exit 1 + fi + + user="$1" + password="$2" + host=$(get_host $user) + + mysql -e "SET PASSWORD FOR '$user'@'$host' = PASSWORD('$password');" +} + + +# +# Argument processing. +# + +if [ $# -lt 1 ]; then + usage + exit 1 +fi + +command="$1" +shift + +case "$command" in + list) + list $@ + ;; + passwd) + passwd $@ + ;; + *) + error "Unknown command: $command." + ;; +esac diff --git a/mysql/files/xinetd/mysqlchk b/mysql/files/xinetd/mysqlchk new file mode 100644 index 00000000..d7c12935 --- /dev/null +++ b/mysql/files/xinetd/mysqlchk @@ -0,0 +1,13 @@ +# Ansible managed +service mysqlchk +{ + socket_type = stream + port = 8306 + protocol = tcp + wait = no + type = UNLISTED + user = root + server = /usr/share/scripts/mysqlchk.sh + log_on_failure += USERID + disable = no +} diff --git a/mysql/files/xinetd/mysqlchk.sh b/mysql/files/xinetd/mysqlchk.sh new file mode 100644 index 00000000..7b5860d2 --- /dev/null +++ b/mysql/files/xinetd/mysqlchk.sh @@ -0,0 +1,54 @@ +#!/bin/sh + +# Ansible managed +# +# http://sysbible.org/x/2008/12/04/having-haproxy-check-mysql-status-through-a-xinetd-script/ +# +# This script checks if a mysql server is healthy running on localhost. It will +# return: +# +# "HTTP/1.x 200 OK\r" (if mysql is running smoothly) +# +# - OR - +# +# "HTTP/1.x 500 Internal Server Error\r" (else) +# +# The purpose of this script is make haproxy capable of monitoring mysql properly +# +# Author: Unai Rodriguez +# +# It is recommended that a low-privileged-mysql user is created to be used by +# this script. Something like this: +# +# mysql> GRANT SELECT on mysql.* TO 'mysqlchkusr'@'localhost' \ +# -> IDENTIFIED BY '257retfg2uysg218' WITH GRANT OPTION; +# mysql> flush privileges; + +TMP_FILE="/tmp/mysqlchk.out" +ERR_FILE="/tmp/mysqlchk.err" + +# +# We perform a simple query that should return a few results :-p +# +/usr/bin/mysql --defaults-file=/etc/mysql/debian.cnf -e "show databases;" > $TMP_FILE 2> $ERR_FILE + +# +# Check the output. If it is not empty then everything is fine and we return +# something. Else, we just do not return anything. +# + +if [ "$(/bin/cat $TMP_FILE)" != "" ]; then + # mysql is fine, return http 200 + /bin/echo -e "HTTP/1.1 200 OK\r\n" + /bin/echo -e "Content-Type: Content-Type: text/plain\r\n" + /bin/echo -e "\r\n" + /bin/echo -e "MySQL is running.\r\n" + /bin/echo -e "\r\n" +else + # mysql is fine, return http 503 + /bin/echo -e "HTTP/1.1 503 Service Unavailable\r\n" + /bin/echo -e "Content-Type: Content-Type: text/plain\r\n" + /bin/echo -e "\r\n" + /bin/echo -e "MySQL is *down*.\r\n" + /bin/echo -e "\r\n" +fi diff --git a/mysql/handlers/main.yml b/mysql/handlers/main.yml index 2ea13151..50755f30 100644 --- a/mysql/handlers/main.yml +++ b/mysql/handlers/main.yml @@ -23,3 +23,8 @@ systemd: name: mysql daemon_reload: yes + +- name: 'restart xinetd' + service: + name: 'xinetd' + state: 'restart' diff --git a/mysql/tasks/main.yml b/mysql/tasks/main.yml index 89ee6866..11435c73 100644 --- a/mysql/tasks/main.yml +++ b/mysql/tasks/main.yml @@ -22,6 +22,9 @@ - include: config_jessie.yml when: ansible_distribution_release == "jessie" +- include: replication.yml + when: mysql_replication + - include: datadir.yml - include: logdir.yml diff --git a/mysql/tasks/replication.yml b/mysql/tasks/replication.yml new file mode 100644 index 00000000..6e5ee039 --- /dev/null +++ b/mysql/tasks/replication.yml @@ -0,0 +1,53 @@ +--- + +- name: 'Copy MySQL configuration for replication' + template: + src: 'replication.cnf.j2' + dest: "{{ mysql_config_directory }}/zzzz-replication.cnf" + with_first_found: + - "templates/mysql/replication.{{ inventory_hostname }}.cnf.j2" + - "templates/mysql/replication.{{ host_group }}.cnf.j2" + - 'templates/mysql/replication.cnf.j2' + - 'replication.cnf.j2' + notify: 'restart mysql' + +- name: 'Create repl user' + mysql_user: + name: 'repl' + host: '%' + encrypted: true + password: "{{ mysql_repl_password }}" + priv: '*.*:REPLICATION SLAVE,REPLICATION CLIENT' + update_password: 'on_create' + state: 'present' + register: create_repl_user + when: mysql_repl_password | length > 0 + +- name: 'Add Nagios check for replication' + template: + src: 'replication_check.cfg.j2' + dest: '/etc/nagios/nrpe.d/replication.cfg' + notify: 'restart nagios-nrpe-server' + +- name: 'Install xinetd' + apt: + name: 'xinetd' + +- name: 'Add xinetd configuration for MySQL HAProxy check' + copy: + src: 'xinetd/mysqlchk' + dest: '/etc/xinetd.d/' + mode: '0644' + notify: 'restart xinetd' + +- name: 'Copy mysqlchk script' + copy: + src: 'xinetd/mysqlchk.sh' + dest: '/usr/share/scripts/' + mode: '0755' + +- name: 'Copy dbadmin script' + copy: + src: 'dbadmin.sh' + dest: '/usr/share/scripts/' + mode: '0755' diff --git a/mysql/templates/evolinux-custom.cnf.j2 b/mysql/templates/evolinux-custom.cnf.j2 index f8ee104e..fd50fb36 100644 --- a/mysql/templates/evolinux-custom.cnf.j2 +++ b/mysql/templates/evolinux-custom.cnf.j2 @@ -29,9 +29,4 @@ query_cache_limit = {{ mysql_query_cache_limit }} {% if mysql_query_cache_limit %} query_cache_size = {{ mysql_query_cache_size }} {% endif %} -{% if mysql_log_bin %} -log_bin = {{ mysql_log_bin }} -{% endif %} -{% if mysql_server_id %} -server_id = {{ mysql_server_id }} -{% endif %} + diff --git a/mysql/templates/replication.cnf.j2 b/mysql/templates/replication.cnf.j2 new file mode 100644 index 00000000..f6da45d9 --- /dev/null +++ b/mysql/templates/replication.cnf.j2 @@ -0,0 +1,7 @@ +# {{ansible_managed}} + +[mysqld] +{% if mysql_log_bin %} +log_bin = {{ mysql_log_bin }} +{% endif %} +server_id = {{ mysql_server_id }} diff --git a/mysql/templates/replication_check.cfg.j2 b/mysql/templates/replication_check.cfg.j2 new file mode 100644 index 00000000..76135811 --- /dev/null +++ b/mysql/templates/replication_check.cfg.j2 @@ -0,0 +1,3 @@ +# ansible managed + +command[check_mysql_slave]=/usr/lib/nagios/plugins/check_mysql --check-slave -H localhost -f ~nagios/.my.cnf -w 1800 -c 3600 From df99e9ac4fd3afec091c7c74851132ad3d3ca3af Mon Sep 17 00:00:00 2001 From: Patrick Marchand Date: Fri, 18 Oct 2019 10:40:14 -0400 Subject: [PATCH 07/88] Make it possible for mysql role to copy evolix scripts Based myself off of the webapps/evoadmin-web role, but I'm not sure we still consider this a hack or not. We use a read only /usr fs, so we need to remount it to add scripts in /usr/local/share. --- mysql/tasks/replication.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mysql/tasks/replication.yml b/mysql/tasks/replication.yml index 6e5ee039..65939ba7 100644 --- a/mysql/tasks/replication.yml +++ b/mysql/tasks/replication.yml @@ -40,6 +40,10 @@ mode: '0644' notify: 'restart xinetd' +# /!\ Warning, this is a temporary hack +- include_role: + name: remount-usr + - name: 'Copy mysqlchk script' copy: src: 'xinetd/mysqlchk.sh' From c39f63048c65b9591e5521b9c8c9fab1d033f48a Mon Sep 17 00:00:00 2001 From: Patrick Marchand Date: Wed, 4 Dec 2019 10:35:49 -0500 Subject: [PATCH 08/88] Removed useless dbadmin script from mysql replication tasks It was used by a very specific client case and is not needed for a general role. --- mysql/files/dbadmin.sh | 101 ------------------------------------ mysql/tasks/replication.yml | 6 --- 2 files changed, 107 deletions(-) delete mode 100644 mysql/files/dbadmin.sh diff --git a/mysql/files/dbadmin.sh b/mysql/files/dbadmin.sh deleted file mode 100644 index f5e61ea8..00000000 --- a/mysql/files/dbadmin.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/sh -# -# Manage MySQL accounts and databases. -# -# Note: in the following code: -# - account means user@host -# - user is the user part of account -# - -MYSQL_OPTS="--raw --skip-column-names --skip-line-numbers" - -usage() { - cat <&2 -Usage: $0 [] - -Available commands are: - - list [] - List all accounts and their databases, separated by semi-colon. If user - is specified, list databases for this user only. - - passwd - Change password for specified user. - -EOT -} - -error() { - printf >&2 "Error: $@\n" -} - -get_host() { - user="$1" - host=$(mysql $MYSQL_OPTS --execute "SELECT host FROM mysql.user WHERE user='$user'") - if [ $(echo "$host" |wc -l) -gt 1 ]; then - # TODO: Not perfect! - echo "$host" |grep '%' - else - echo $host - fi -} - -get_dbs() { - account="$1" - echo "$(mysql $MYSQL_OPTS --execute "SHOW GRANTS FOR $account" |perl -ne 'print "$1 " if (/^GRANT (?!USAGE).* ON `(.*)`/)')" -} - -get_accounts() { - echo "$(mysql $MYSQL_OPTS --execute "SELECT user,host FROM mysql.user;" |perl -ne 'print "$1\@$2\n" if (/^([^\s]+)\s+([^\s]+)$/)'|sed "s/^/'/; s/@/'@'/; s/$/'/;")" -} - -list() { - if [ $# -gt 0 ]; then - user="$1" - host=$(get_host $user) - account="'$user'@'$host'" - echo $account:$(get_dbs "$account") - else - for account in $(get_accounts); do - echo $account:$(get_dbs "$account") - done - fi -} - -passwd() { - if [ $# -ne 2 ]; then - usage - exit 1 - fi - - user="$1" - password="$2" - host=$(get_host $user) - - mysql -e "SET PASSWORD FOR '$user'@'$host' = PASSWORD('$password');" -} - - -# -# Argument processing. -# - -if [ $# -lt 1 ]; then - usage - exit 1 -fi - -command="$1" -shift - -case "$command" in - list) - list $@ - ;; - passwd) - passwd $@ - ;; - *) - error "Unknown command: $command." - ;; -esac diff --git a/mysql/tasks/replication.yml b/mysql/tasks/replication.yml index 65939ba7..33263815 100644 --- a/mysql/tasks/replication.yml +++ b/mysql/tasks/replication.yml @@ -49,9 +49,3 @@ src: 'xinetd/mysqlchk.sh' dest: '/usr/share/scripts/' mode: '0755' - -- name: 'Copy dbadmin script' - copy: - src: 'dbadmin.sh' - dest: '/usr/share/scripts/' - mode: '0755' From 8778d69102eae569a3085e5a13b0712a2ce3b6f7 Mon Sep 17 00:00:00 2001 From: Patrick Marchand Date: Wed, 4 Dec 2019 11:11:48 -0500 Subject: [PATCH 09/88] Removed redundant nagios checks They are already installed by the base roles. --- mysql/handlers/main.yml | 5 ----- mysql/tasks/replication.yml | 6 ------ mysql/templates/replication_check.cfg.j2 | 3 --- 3 files changed, 14 deletions(-) delete mode 100644 mysql/templates/replication_check.cfg.j2 diff --git a/mysql/handlers/main.yml b/mysql/handlers/main.yml index 50755f30..87a7613a 100644 --- a/mysql/handlers/main.yml +++ b/mysql/handlers/main.yml @@ -4,11 +4,6 @@ name: munin-node state: restarted -- name: restart nagios-nrpe-server - service: - name: nagios-nrpe-server - state: restarted - - name: restart mysql service: name: mysql diff --git a/mysql/tasks/replication.yml b/mysql/tasks/replication.yml index 33263815..54f5e3e9 100644 --- a/mysql/tasks/replication.yml +++ b/mysql/tasks/replication.yml @@ -23,12 +23,6 @@ register: create_repl_user when: mysql_repl_password | length > 0 -- name: 'Add Nagios check for replication' - template: - src: 'replication_check.cfg.j2' - dest: '/etc/nagios/nrpe.d/replication.cfg' - notify: 'restart nagios-nrpe-server' - - name: 'Install xinetd' apt: name: 'xinetd' diff --git a/mysql/templates/replication_check.cfg.j2 b/mysql/templates/replication_check.cfg.j2 deleted file mode 100644 index 76135811..00000000 --- a/mysql/templates/replication_check.cfg.j2 +++ /dev/null @@ -1,3 +0,0 @@ -# ansible managed - -command[check_mysql_slave]=/usr/lib/nagios/plugins/check_mysql --check-slave -H localhost -f ~nagios/.my.cnf -w 1800 -c 3600 From 18878d58e8e60167f3f920a0dc14013168fd6f05 Mon Sep 17 00:00:00 2001 From: Patrick Marchand Date: Mon, 13 Jan 2020 17:22:16 +0100 Subject: [PATCH 10/88] Replication should set a binlog format This could possible be better served in the base config file, but for now I'll keep it here. --- mysql/defaults/main.yml | 3 ++- mysql/templates/replication.cnf.j2 | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mysql/defaults/main.yml b/mysql/defaults/main.yml index f364de18..501b2cb2 100644 --- a/mysql/defaults/main.yml +++ b/mysql/defaults/main.yml @@ -45,6 +45,7 @@ mysql_restart_if_needed: True # replication variables: mysql_replication: false mysql_log_bin: null +mysql_binlog_format: mixed mysql_server_id: null mysql_bind_address: null -mysql_repl_password: '' \ No newline at end of file +mysql_repl_password: '' diff --git a/mysql/templates/replication.cnf.j2 b/mysql/templates/replication.cnf.j2 index f6da45d9..030f2470 100644 --- a/mysql/templates/replication.cnf.j2 +++ b/mysql/templates/replication.cnf.j2 @@ -5,3 +5,4 @@ log_bin = {{ mysql_log_bin }} {% endif %} server_id = {{ mysql_server_id }} +binlog_format = {{ mysql_binlog_format }} From c142af69f07dc75b75b0fcc72eaff00a9aa909fa Mon Sep 17 00:00:00 2001 From: Patrick Marchand Date: Mon, 13 Jan 2020 17:23:34 +0100 Subject: [PATCH 11/88] Added some mysql variables and allowed forcing config update The default behaviour is kept, but this way we can manage a mysql installation from ansible. --- mysql/defaults/main.yml | 2 ++ mysql/tasks/config_jessie.yml | 2 +- mysql/tasks/config_stretch.yml | 2 +- mysql/templates/evolinux-custom.cnf.j2 | 7 +++++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/mysql/defaults/main.yml b/mysql/defaults/main.yml index 501b2cb2..8544daef 100644 --- a/mysql/defaults/main.yml +++ b/mysql/defaults/main.yml @@ -28,6 +28,8 @@ mysql_tmp_table_size: Null mysql_max_heap_table_size: Null mysql_query_cache_limit: Null mysql_query_cache_size: Null +mysql_max_allowed_packet: Null +mysql_force_custom_config: 'no' mysql_cron_optimize: True mysql_cron_optimize_frequency: weekly diff --git a/mysql/tasks/config_jessie.yml b/mysql/tasks/config_jessie.yml index 9fe11bb7..a5dd4d77 100644 --- a/mysql/tasks/config_jessie.yml +++ b/mysql/tasks/config_jessie.yml @@ -21,6 +21,6 @@ owner: root group: root mode: "0644" - force: no + force: "{{ mysql_force_custom_config }}" tags: - mysql diff --git a/mysql/tasks/config_stretch.yml b/mysql/tasks/config_stretch.yml index 0725ee1f..d6d59efd 100644 --- a/mysql/tasks/config_stretch.yml +++ b/mysql/tasks/config_stretch.yml @@ -21,7 +21,7 @@ owner: root group: root mode: "0644" - force: no + force: "{{ mysql_force_custom_config }}" tags: - mysql diff --git a/mysql/templates/evolinux-custom.cnf.j2 b/mysql/templates/evolinux-custom.cnf.j2 index fd50fb36..daa70a48 100644 --- a/mysql/templates/evolinux-custom.cnf.j2 +++ b/mysql/templates/evolinux-custom.cnf.j2 @@ -29,4 +29,11 @@ query_cache_limit = {{ mysql_query_cache_limit }} {% if mysql_query_cache_limit %} query_cache_size = {{ mysql_query_cache_size }} {% endif %} +{% if mysql_max_allowed_packet %} +max_allowed_packet = {{ mysql_max_allowed_packet }} +{% endif %} +{% if mysql_lower_case_table_names %} +lower_case_table_names = {{ mysql_lower_case_table_names }} +{% endif %} + From e713ad024b5e5f4fb8ddff8bedd8ee2ac52c219b Mon Sep 17 00:00:00 2001 From: Patrick Marchand Date: Mon, 13 Jan 2020 17:50:24 +0100 Subject: [PATCH 12/88] Fix right problem in mysql replication The configuration file was not set to 0644, which caused the file to be ignored by mysql and it's configuration not to be set. --- mysql/tasks/replication.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mysql/tasks/replication.yml b/mysql/tasks/replication.yml index 54f5e3e9..f447d099 100644 --- a/mysql/tasks/replication.yml +++ b/mysql/tasks/replication.yml @@ -4,11 +4,7 @@ template: src: 'replication.cnf.j2' dest: "{{ mysql_config_directory }}/zzzz-replication.cnf" - with_first_found: - - "templates/mysql/replication.{{ inventory_hostname }}.cnf.j2" - - "templates/mysql/replication.{{ host_group }}.cnf.j2" - - 'templates/mysql/replication.cnf.j2' - - 'replication.cnf.j2' + mode: "0644" notify: 'restart mysql' - name: 'Create repl user' From 004b48eee7180fc2b9b99f8482fef840e637dddb Mon Sep 17 00:00:00 2001 From: Patrick Marchand Date: Wed, 15 Jan 2020 15:58:29 +0100 Subject: [PATCH 13/88] Added mysql_innodb_log_file_size option to the mysql role Makes it possible to have larger binary entries when replicating. --- mysql/defaults/main.yml | 2 ++ mysql/templates/evolinux-custom.cnf.j2 | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mysql/defaults/main.yml b/mysql/defaults/main.yml index 8544daef..49115ee9 100644 --- a/mysql/defaults/main.yml +++ b/mysql/defaults/main.yml @@ -30,6 +30,7 @@ mysql_query_cache_limit: Null mysql_query_cache_size: Null mysql_max_allowed_packet: Null mysql_force_custom_config: 'no' +mysql_innodb_log_file_size: Null mysql_cron_optimize: True mysql_cron_optimize_frequency: weekly @@ -51,3 +52,4 @@ mysql_binlog_format: mixed mysql_server_id: null mysql_bind_address: null mysql_repl_password: '' + diff --git a/mysql/templates/evolinux-custom.cnf.j2 b/mysql/templates/evolinux-custom.cnf.j2 index daa70a48..d33a4f2b 100644 --- a/mysql/templates/evolinux-custom.cnf.j2 +++ b/mysql/templates/evolinux-custom.cnf.j2 @@ -35,5 +35,6 @@ max_allowed_packet = {{ mysql_max_allowed_packet }} {% if mysql_lower_case_table_names %} lower_case_table_names = {{ mysql_lower_case_table_names }} {% endif %} - - +{% if mysql_innodb_log_file_size %} +innodb_log_file_size = {{ mysql_innodb_log_file_size }} +{% endif %} From 0a4aeea4818b3c3e66c4eb314eba012244fd5a27 Mon Sep 17 00:00:00 2001 From: Patrick Marchand Date: Mon, 1 Jun 2020 12:03:23 -0400 Subject: [PATCH 14/88] Add option to make a mysql install read only Rebased on unstable --- CHANGELOG.md | 5 +++-- mysql/defaults/main.yml | 2 +- mysql/templates/evolinux-custom.cnf.j2 | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab0d0ced..eee08070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ The **patch** part changes incrementally at each release. ## [Unreleased] ### Added +* mysql: activate binary logs by specifying log_bin path +* mysql: specify a custom server_id +* mysql: option to define as read only ### Changed @@ -47,8 +50,6 @@ The **patch** part changes incrementally at each release. * minifirewall: add a variable to force the check scripts update * mongodb: mongodb: compatibility with Debian 10 * mysql-oracle: backport tasks from mysql role -* mysql: activate binary logs by specifying log_bin path -* mysql: specify a custom server_id * networkd-to-ifconfig: add variables for configuration by variables * packweb-apache: Deploy opcache.php to give some insights on PHP's opcache status * php: variable to install the mysqlnd module instead of the default mysql module diff --git a/mysql/defaults/main.yml b/mysql/defaults/main.yml index 49115ee9..8d8771fa 100644 --- a/mysql/defaults/main.yml +++ b/mysql/defaults/main.yml @@ -52,4 +52,4 @@ mysql_binlog_format: mixed mysql_server_id: null mysql_bind_address: null mysql_repl_password: '' - +mysql_read_only: 0 diff --git a/mysql/templates/evolinux-custom.cnf.j2 b/mysql/templates/evolinux-custom.cnf.j2 index d33a4f2b..63d7ea2d 100644 --- a/mysql/templates/evolinux-custom.cnf.j2 +++ b/mysql/templates/evolinux-custom.cnf.j2 @@ -38,3 +38,4 @@ lower_case_table_names = {{ mysql_lower_case_table_names }} {% if mysql_innodb_log_file_size %} innodb_log_file_size = {{ mysql_innodb_log_file_size }} {% endif %} +read_only = {{ mysql_read_only }} From f034e93b6ec8d98e63b6a00267292cc20aaebab5 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 26 May 2020 14:41:42 +0200 Subject: [PATCH 15/88] whitespaces --- haproxy/templates/haproxy.default.cfg.j2 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/haproxy/templates/haproxy.default.cfg.j2 b/haproxy/templates/haproxy.default.cfg.j2 index 0069d20a..5e060629 100644 --- a/haproxy/templates/haproxy.default.cfg.j2 +++ b/haproxy/templates/haproxy.default.cfg.j2 @@ -22,12 +22,14 @@ global defaults log global + mode http option httplog option dontlognull - timeout connect 5000 - timeout client 50000 - timeout server 50000 + timeout connect 5000 + timeout client 50000 + timeout server 50000 + errorfile 400 /etc/haproxy/errors/400.http errorfile 403 /etc/haproxy/errors/403.http errorfile 408 /etc/haproxy/errors/408.http From 66b62c56294d7ccc47491e289bdc07f7480a77e6 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 26 May 2020 14:42:35 +0200 Subject: [PATCH 16/88] haproxy: chroot and socket path are configurable --- CHANGELOG.md | 1 + haproxy/defaults/main.yml | 3 +++ haproxy/templates/haproxy.default.cfg.j2 | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eee08070..a5c6151c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The **patch** part changes incrementally at each release. ### Changed * packweb-apache: Don't turn on mod-evasive emails by default +* haproxy: chroot and socket path are configurable ### Fixed diff --git a/haproxy/defaults/main.yml b/haproxy/defaults/main.yml index cfe3fcea..a0f4f259 100644 --- a/haproxy/defaults/main.yml +++ b/haproxy/defaults/main.yml @@ -4,3 +4,6 @@ haproxy_backports: "{{ haproxy_jessie_backports | default(false, true) }}" haproxy_stats_url: "http://127.0.0.1:8080/" haproxy_update_config: True haproxy_force_config: True + +haproxy_socket: /run/haproxy/admin.sock +haproxy_chroot: /var/lib/haproxy diff --git a/haproxy/templates/haproxy.default.cfg.j2 b/haproxy/templates/haproxy.default.cfg.j2 index 5e060629..583cb347 100644 --- a/haproxy/templates/haproxy.default.cfg.j2 +++ b/haproxy/templates/haproxy.default.cfg.j2 @@ -3,8 +3,8 @@ global log /dev/log local0 log /dev/log local1 notice - chroot /var/lib/haproxy - stats socket /run/haproxy/admin.sock mode 660 level admin + chroot {{ haproxy_chroot }} + stats socket {{ haproxy_socket }} mode 660 level admin stats timeout 30s user haproxy group haproxy From bd8a30a107415b79a89c2c8ce13085097e26454e Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 2 Jun 2020 10:54:48 +0200 Subject: [PATCH 17/88] redis: create sudoers file if missing --- CHANGELOG.md | 1 + redis/tasks/nrpe.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5c6151c..5fa12446 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The **patch** part changes incrementally at each release. * packweb-apache: Don't turn on mod-evasive emails by default * haproxy: chroot and socket path are configurable +* redis: create sudoers file if missing ### Fixed diff --git a/redis/tasks/nrpe.yml b/redis/tasks/nrpe.yml index f499ed2f..a93c21af 100644 --- a/redis/tasks/nrpe.yml +++ b/redis/tasks/nrpe.yml @@ -50,6 +50,7 @@ line: 'nagios ALL = NOPASSWD: {{ redis_check_redis_path }}' insertafter: '^nagios' validate: "visudo -cf %s" + create: yes when: redis_instance_name is defined tags: - redis From 107f1f06e6783f664827386fbafe37693fe1d25d Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 2 Jun 2020 10:55:35 +0200 Subject: [PATCH 18/88] redis: new syntax for match filter --- CHANGELOG.md | 1 + redis/templates/redis-server@stretch.service.j2 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fa12446..bc73a31a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The **patch** part changes incrementally at each release. * packweb-apache: Don't turn on mod-evasive emails by default * haproxy: chroot and socket path are configurable * redis: create sudoers file if missing +* redis: new syntax for match filter ### Fixed diff --git a/redis/templates/redis-server@stretch.service.j2 b/redis/templates/redis-server@stretch.service.j2 index ea29da5e..3f14a296 100644 --- a/redis/templates/redis-server@stretch.service.j2 +++ b/redis/templates/redis-server@stretch.service.j2 @@ -22,7 +22,7 @@ UMask=007 PrivateTmp=yes LimitNOFILE=65535 PrivateDevices=yes -ProtectHome={{ redis_data_dir_prefix | match('/home') | ternary('no', 'yes') }} +ProtectHome={{ redis_data_dir_prefix is match('/home') | ternary('no', 'yes') }} ReadOnlyDirectories=/ ReadWriteDirectories=-{{ redis_data_dir_prefix }}-%i ReadWriteDirectories=-{{ redis_log_dir_prefix }}-%i From 291ad2ba40cbb244c94433c53fabfb3e1d7263fd Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 2 Jun 2020 11:22:56 +0200 Subject: [PATCH 19/88] redis: raise an error is port 6379 is used in "instance" mode --- CHANGELOG.md | 1 + redis/tasks/instance-server.yml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc73a31a..95496976 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The **patch** part changes incrementally at each release. * haproxy: chroot and socket path are configurable * redis: create sudoers file if missing * redis: new syntax for match filter +* redis: raise an error is port 6379 is used in "instance" mode ### Fixed diff --git a/redis/tasks/instance-server.yml b/redis/tasks/instance-server.yml index e87c6ce9..6437567c 100644 --- a/redis/tasks/instance-server.yml +++ b/redis/tasks/instance-server.yml @@ -1,5 +1,11 @@ --- +- name: Verify Redis port + assert: + that: + - redis_port != 6379 + msg: "If you want to use port 6379, use the default instance, not a named instance." + - name: "Instance '{{ redis_instance_name }}' group is present" group: name: "redis-{{ redis_instance_name }}" From 6d31568a4123eb12dfcecbc051080e5f8945d7ec Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 3 Jun 2020 12:06:03 +0200 Subject: [PATCH 20/88] evoacme: upstream release 20.06 --- CHANGELOG.md | 1 + evoacme/files/evoacme.sh | 21 ++++++++++----------- evoacme/files/make-csr.sh | 2 +- evoacme/files/vhost-domains.sh | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95496976..3c782fe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The **patch** part changes incrementally at each release. * redis: create sudoers file if missing * redis: new syntax for match filter * redis: raise an error is port 6379 is used in "instance" mode +* evoacme: upstream release 20.06 ### Fixed diff --git a/evoacme/files/evoacme.sh b/evoacme/files/evoacme.sh index 2009bd22..3b6d9771 100755 --- a/evoacme/files/evoacme.sh +++ b/evoacme/files/evoacme.sh @@ -118,21 +118,21 @@ main() { [ "$1" = "-V" ] || [ "$1" = "--version" ] && show_version && exit 0 mkdir -p "${ACME_DIR}" - chown acme: "${ACME_DIR}" + chown root: "${ACME_DIR}" [ -w "${ACME_DIR}" ] || error "Directory ${ACME_DIR} is not writable" [ -d "${CSR_DIR}" ] || error "Directory ${CSR_DIR} is not found" mkdir -p "${CRT_DIR}" - chown acme: "${CRT_DIR}" + chown root: "${CRT_DIR}" [ -w "${CRT_DIR}" ] || error "Directory ${CRT_DIR} is not writable" mkdir -p "${LOG_DIR}" - chown acme: "${LOG_DIR}" + chown root: "${LOG_DIR}" [ -w "${LOG_DIR}" ] || error "Directory ${LOG_DIR} is not writable" mkdir -p "${HOOKS_DIR}" - chown acme: "${HOOKS_DIR}" + chown root: "${HOOKS_DIR}" [ -d "${HOOKS_DIR}" ] || error "Directory ${HOOKS_DIR} is not found" readonly VHOST=$(basename "$1" .conf) @@ -195,7 +195,7 @@ main() { [ -d "${NEW_DIR}" ] && error "${NEW_DIR} directory already exists, remove it manually." mkdir -p "${NEW_DIR}" - chown -R acme: "${CRT_DIR}" + chown -R root: "${CRT_DIR}" chmod -R 0700 "${CRT_DIR}" chmod -R g+rX "${CRT_DIR}" debug "New cert will be created in ${NEW_DIR}" @@ -218,15 +218,14 @@ main() { CERTBOT_REGISTRATION="${CERTBOT_REGISTRATION} --register-unsafely-without-email" fi - # Permissions checks for acme user - sudo -u acme test -r "${CSR_FILE}" || error "File ${CSR_FILE} is not readable by user 'acme'" - sudo -u acme test -w "${NEW_DIR}" || error "Directory ${NEW_DIR} is not writable by user 'acme'" + # Permissions checks + test -r "${CSR_FILE}" || error "File ${CSR_FILE} is not readable" + test -w "${NEW_DIR}" || error "Directory ${NEW_DIR} is not writable" # create a certificate with certbot # we disable the set -e during the certbot call set +e - sudo -u acme \ - "${CERTBOT_BIN}" \ + "${CERTBOT_BIN}" \ certonly \ ${CERTBOT_MODE} \ ${CERTBOT_REGISTRATION} \ @@ -304,7 +303,7 @@ readonly QUIET=${QUIET:-"0"} readonly TEST=${TEST:-"0"} readonly DRY_RUN=${DRY_RUN:-"0"} -readonly VERSION="19.11" +readonly VERSION="20.06" # Read configuration file, if it exists [ -r /etc/default/evoacme ] && . /etc/default/evoacme diff --git a/evoacme/files/make-csr.sh b/evoacme/files/make-csr.sh index b61d5adb..d7abb204 100755 --- a/evoacme/files/make-csr.sh +++ b/evoacme/files/make-csr.sh @@ -265,7 +265,7 @@ readonly ARGS=$@ readonly VERBOSE=${VERBOSE:-"0"} readonly QUIET=${QUIET:-"0"} -readonly VERSION="19.11" +readonly VERSION="20.06" # Read configuration file, if it exists [ -r /etc/default/evoacme ] && . /etc/default/evoacme diff --git a/evoacme/files/vhost-domains.sh b/evoacme/files/vhost-domains.sh index d2f9c61c..6bf91286 100755 --- a/evoacme/files/vhost-domains.sh +++ b/evoacme/files/vhost-domains.sh @@ -170,7 +170,7 @@ readonly ARGS=$@ readonly VERBOSE=${VERBOSE:-"0"} readonly QUIET=${QUIET:-"0"} -readonly VERSION="19.11" +readonly VERSION="20.06" readonly SRV_IP=${SRV_IP:-""} From 35123f8189629582ab972da0beabd65af91ddf60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Thu, 4 Jun 2020 16:50:35 +0900 Subject: [PATCH 21/88] evolinux-base: Add check_hpraid.sh This script is meant to be executed as a cron by executing Nagios NRPE plugin check_hpraid and notify by mail any errors --- evolinux-base/files/check_hpraid.sh | 64 +++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 evolinux-base/files/check_hpraid.sh diff --git a/evolinux-base/files/check_hpraid.sh b/evolinux-base/files/check_hpraid.sh new file mode 100644 index 00000000..b4c8ffc0 --- /dev/null +++ b/evolinux-base/files/check_hpraid.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script is meant to be executed as a cron by executing Nagios +# NRPE plugin check_hpraid and notify by mail any errors + +TMPDIR=/tmp +md5sum=$(command -v md5sum) +awk=$(command -v awk) +check_hpraid="/usr/local/lib/nagios/plugins/check_hpraid -v" +check_hpraid_output=$(mktemp -p $TMPDIR check_hpraid_XXX) +check_hpraid_last="$TMPDIR/check_hpraid_last" +trap trapFunc EXIT ERR + +testDeps() { + + test -x "$md5sum" || (echo "md5sum binary not found"; exit 1) + test -x "$awk" || (echo "awk binary not found"; exit 1) +} + +main() { + + if ! $check_hpraid > "$check_hpraid_output"; then + error=true + else + error=false + fi + if [ ! -f $check_hpraid_last ]; then + cp "$check_hpraid_output" $check_hpraid_last + fi + + # If output and last check is different, display differences and + # exit + md5_now=$(md5sum "$check_hpraid_output" | awk '{print $1}') + md5_last=$(md5sum $check_hpraid_last | awk '{print $1}') + if [[ "$md5_now" != "$md5_last" ]]; then + cat << EOT + Different RAID state detected. + Was: + $check_hpraid_last + Is now: + $check_hpraid_output +EOT + exit 1 + fi + + # If check_hpraid returned error, display output, save status and + # exit + if $error; then + cp "$check_hpraid_output" $check_hpraid_last + cat "$check_hpraid_output" + exit 1 + else + exit 0 + fi +} + +trapFunc() { + + rm "$check_hpraid_output" +} + +testDeps +main From f963eaacf69557a26384c83f97d8ceb1e97f4a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Thu, 4 Jun 2020 17:23:14 +0900 Subject: [PATCH 22/88] evolinux-base: check_hpraid.sh: Fix RAID state detection --- evolinux-base/files/check_hpraid.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/evolinux-base/files/check_hpraid.sh b/evolinux-base/files/check_hpraid.sh index b4c8ffc0..3c266ba4 100644 --- a/evolinux-base/files/check_hpraid.sh +++ b/evolinux-base/files/check_hpraid.sh @@ -37,11 +37,12 @@ main() { cat << EOT Different RAID state detected. Was: - $check_hpraid_last + $(cat $check_hpraid_last) Is now: - $check_hpraid_output + $(cat $check_hpraid_output) EOT - exit 1 + cp "$check_hpraid_output" $check_hpraid_last + exit 1 fi # If check_hpraid returned error, display output, save status and From bdb234c0cdd571477b3fd5ba241bae2681872ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Thu, 4 Jun 2020 17:32:49 +0900 Subject: [PATCH 23/88] evolinux-base: check_hpraid.sh: Fix missing copy of RAID state --- evolinux-base/files/check_hpraid.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/evolinux-base/files/check_hpraid.sh b/evolinux-base/files/check_hpraid.sh index 3c266ba4..9c5548c2 100644 --- a/evolinux-base/files/check_hpraid.sh +++ b/evolinux-base/files/check_hpraid.sh @@ -52,6 +52,7 @@ EOT cat "$check_hpraid_output" exit 1 else + cp "$check_hpraid_output" $check_hpraid_last exit 0 fi } From 877a32bbf06ce6cdb063f07cfee4da30fd4c5f9b Mon Sep 17 00:00:00 2001 From: Jeremy Dubois Date: Thu, 4 Jun 2020 10:43:17 +0200 Subject: [PATCH 24/88] nagios-nrpe / evolinux-base : new ntp server variable Online hosted servers must use ntp.online.net as ntp server, because others one are rate limited. Default ntp server is pool.ntp.org, and a custom one can be set with the nagios_nrpe_ntp_server variable. --- evolinux-base/tasks/provider_online.yml | 4 ++++ nagios-nrpe/defaults/main.yml | 3 +++ nagios-nrpe/templates/evolix.cfg.j2 | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/evolinux-base/tasks/provider_online.yml b/evolinux-base/tasks/provider_online.yml index 76f31e4f..8174d15c 100644 --- a/evolinux-base/tasks/provider_online.yml +++ b/evolinux-base/tasks/provider_online.yml @@ -1,4 +1,8 @@ - debug: msg: "Online DNS servers fails sometimes! Please change them in /etc/resolv.conf." +- name: custom NTP server for Online servers + set_fact: + nagios_nrpe_default_ntp_server: "ntp.online.net" + # - meta: flush_handlers diff --git a/nagios-nrpe/defaults/main.yml b/nagios-nrpe/defaults/main.yml index cb2f901e..ba519361 100644 --- a/nagios-nrpe/defaults/main.yml +++ b/nagios-nrpe/defaults/main.yml @@ -4,9 +4,12 @@ nagios_nrpe_additional_allowed_hosts: [] nagios_nrpe_allowed_hosts: "{{ nagios_nrpe_default_allowed_hosts | union(nagios_nrpe_additional_allowed_hosts) | unique }}" nagios_nrpe_pgsql_passwd: PGSQL_PASSWD nagios_nrpe_amavis_from: "foobar@{{ ansible_domain }}" +nagios_nrpe_default_ntp_server: "pool.ntp.org" +nagios_nrpe_ntp_server: Null nagios_nrpe_force_update_allowed_hosts: False nagios_nrpe_check_proxy_host: "www.example.com" nagios_plugins_directory: "/usr/local/lib/nagios/plugins" + diff --git a/nagios-nrpe/templates/evolix.cfg.j2 b/nagios-nrpe/templates/evolix.cfg.j2 index 89d7e7d9..433a0cd5 100644 --- a/nagios-nrpe/templates/evolix.cfg.j2 +++ b/nagios-nrpe/templates/evolix.cfg.j2 @@ -17,7 +17,7 @@ command[check_users]=/usr/lib/nagios/plugins/check_users -w 5 -c 10 # Generic services checks command[check_smtp]=/usr/lib/nagios/plugins/check_smtp -H localhost command[check_dns]=/usr/lib/nagios/plugins/check_dns -H evolix.net -command[check_ntp]=/usr/lib/nagios/plugins/check_ntp -H ntp2.evolix.net +command[check_ntp]=/usr/lib/nagios/plugins/check_ntp -H {{ nagios_nrpe_ntp_server or nagios_nrpe_default_ntp_server | mandatory }} command[check_ssh]=/usr/lib/nagios/plugins/check_ssh localhost command[check_mailq]=/usr/lib/nagios/plugins/check_mailq -M postfix -w 10 -c 20 From cd29ee4d33be597e29f45da12e44632c22453cfc Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 3 Jun 2020 18:23:54 +0200 Subject: [PATCH 25/88] certbot: install certbot dependencies non-interactively for jessie --- CHANGELOG.md | 1 + certbot/handlers/main.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c782fe4..a92ddc8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The **patch** part changes incrementally at each release. * redis: new syntax for match filter * redis: raise an error is port 6379 is used in "instance" mode * evoacme: upstream release 20.06 +* certbot: install certbot dependencies non-interactively for jessie ### Fixed diff --git a/certbot/handlers/main.yml b/certbot/handlers/main.yml index 4a9d305a..ef3272ee 100644 --- a/certbot/handlers/main.yml +++ b/certbot/handlers/main.yml @@ -20,4 +20,4 @@ daemon_reload: yes - name: install certbot-auto - command: /usr/local/bin/certbot --install-only + command: /usr/local/bin/certbot --noninteractive --install-only From 8d8e5f6998c51c8fe7e2e5f488123b4231d5ff4a Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 3 Jun 2020 18:24:40 +0200 Subject: [PATCH 26/88] certbot: restore compatibility with old Nginx --- CHANGELOG.md | 2 ++ certbot/templates/acme-challenge/nginx.conf.j2 | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a92ddc8c..53feb52e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ The **patch** part changes incrementally at each release. ### Fixed +* certbot: restore compatibility with old Nginx + ### Removed ### Security diff --git a/certbot/templates/acme-challenge/nginx.conf.j2 b/certbot/templates/acme-challenge/nginx.conf.j2 index a0730f3c..e4d3244b 100644 --- a/certbot/templates/acme-challenge/nginx.conf.j2 +++ b/certbot/templates/acme-challenge/nginx.conf.j2 @@ -1,5 +1,9 @@ location ~ /.well-known/acme-challenge { +{% if ansible_distribution == "Debian" and ansible_distribution_major_version is version('8', '<=') %} + alias {{ certbot_work_dir }}/.well-known/acme-challenge; +{% else %} alias {{ certbot_work_dir }}/; +{% endif %} try_files $uri =404; allow all; } From a841e60531488002634579a7aea12a18d492fb0b Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 4 Jun 2020 11:22:52 +0200 Subject: [PATCH 27/88] haproxy: syntax and whitespaces --- haproxy/tasks/main.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/haproxy/tasks/main.yml b/haproxy/tasks/main.yml index 80e846e6..8e3094fb 100644 --- a/haproxy/tasks/main.yml +++ b/haproxy/tasks/main.yml @@ -4,8 +4,8 @@ name: ssl-cert state: present tags: - - haproxy - - packages + - haproxy + - packages - include: packages_backports.yml when: haproxy_backports @@ -15,8 +15,8 @@ name: haproxy state: present tags: - - haproxy - - packages + - haproxy + - packages - name: Copy HAProxy configuration template: @@ -25,14 +25,14 @@ force: "{{ haproxy_force_config }}" validate: "haproxy -c -f %s" with_first_found: - - "templates/haproxy/haproxy.{{ inventory_hostname }}.cfg.j2" - - "templates/haproxy/haproxy.{{ host_group }}.cfg.j2" - - "templates/haproxy/haproxy.default.cfg.j2" - - "haproxy.default.cfg.j2" + - "templates/haproxy/haproxy.{{ inventory_hostname }}.cfg.j2" + - "templates/haproxy/haproxy.{{ host_group }}.cfg.j2" + - "templates/haproxy/haproxy.default.cfg.j2" + - "haproxy.default.cfg.j2" notify: reload haproxy - when: "{{ haproxy_update_config }}" + when: haproxy_update_config tags: - - haproxy - - config + - haproxy + - config - include: munin.yml From 0dfe97d77162d2546173d6aadae8e4c54a4694d4 Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Thu, 4 Jun 2020 11:34:26 +0200 Subject: [PATCH 28/88] packweb-apache: Install zip & unzip by default --- CHANGELOG.md | 1 + packweb-apache/tasks/main.yml | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53feb52e..0d3cb6ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The **patch** part changes incrementally at each release. * mysql: activate binary logs by specifying log_bin path * mysql: specify a custom server_id * mysql: option to define as read only +* packweb-apache: Install zip & unzip by default ### Changed diff --git a/packweb-apache/tasks/main.yml b/packweb-apache/tasks/main.yml index 9c36c888..922dcea8 100644 --- a/packweb-apache/tasks/main.yml +++ b/packweb-apache/tasks/main.yml @@ -5,6 +5,13 @@ when: - ansible_distribution != "Debian" or ansible_distribution_major_version is version('8', '<') +- name: Additional packages are installed + apt: + name: + - zip + - unzip + state: present + - name: install info.php copy: src: info.php From 72939c3e57154064c00b41915c28863f716388c0 Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Thu, 4 Jun 2020 11:39:51 +0200 Subject: [PATCH 29/88] php: Install php-sqlite by default --- CHANGELOG.md | 1 + php/tasks/main_buster.yml | 5 +++-- php/tasks/main_jessie.yml | 1 + php/tasks/main_stretch.yml | 5 +++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d3cb6ba..2b83ee54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The **patch** part changes incrementally at each release. * mysql: specify a custom server_id * mysql: option to define as read only * packweb-apache: Install zip & unzip by default +* php: Install php-sqlite by default ### Changed diff --git a/php/tasks/main_buster.yml b/php/tasks/main_buster.yml index 240f9b54..16eed389 100644 --- a/php/tasks/main_buster.yml +++ b/php/tasks/main_buster.yml @@ -27,6 +27,7 @@ - php-mysql # php-mcrypt is no longer packaged for PHP 7.2 - php-pgsql + - php-sqlite3 - php-gettext - php-curl - php-ssh2 @@ -76,7 +77,7 @@ - include: config_fpm.yml when: php_fpm_enable - + - name: Enforce permissions on PHP fpm directory file: dest: /etc/php/7.3/fpm @@ -85,7 +86,7 @@ - include: config_apache.yml when: php_apache_enable - + - name: Enforce permissions on PHP apache2 directory file: dest: /etc/php/7.3/apache2 diff --git a/php/tasks/main_jessie.yml b/php/tasks/main_jessie.yml index fdeed8c8..8e99280c 100644 --- a/php/tasks/main_jessie.yml +++ b/php/tasks/main_jessie.yml @@ -26,6 +26,7 @@ - php5-mcrypt - "{{ php_modules_mysqlnd | bool | ternary('php5-mysqlnd','php5-mysql') }}" - php5-pgsql + - php5-sqlite - php-gettext - php5-intl - php5-curl diff --git a/php/tasks/main_stretch.yml b/php/tasks/main_stretch.yml index 75f69ab0..136fa346 100644 --- a/php/tasks/main_stretch.yml +++ b/php/tasks/main_stretch.yml @@ -26,6 +26,7 @@ - php-ldap - "{{ php_modules_mysqlnd | bool | ternary('php-mysqlnd','php-mysql') }}" - php-pgsql + - php-sqlite3 # php-mcrypt is no longer packaged for PHP 7.2 - php-gettext - php-curl @@ -77,7 +78,7 @@ - include: config_fpm.yml when: php_fpm_enable - + - name: Enforce permissions on PHP fpm directory file: dest: /etc/php/7.0/fpm @@ -86,7 +87,7 @@ - include: config_apache.yml when: php_apache_enable - + - name: Enforce permissions on PHP apache2 directory file: dest: /etc/php/7.0/apache2 From 193b640226cab9e2f8e4f4bbac7b67683f0eb93e Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Thu, 4 Jun 2020 11:42:17 +0200 Subject: [PATCH 30/88] lxc-php: Install php-sqlite by default --- CHANGELOG.md | 1 + lxc-php/tasks/php56.yml | 2 +- lxc-php/tasks/php70.yml | 2 +- lxc-php/tasks/php73.yml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b83ee54..c9c40fc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The **patch** part changes incrementally at each release. ## [Unreleased] ### Added +* lxc-php: Install php-sqlite by default * mysql: activate binary logs by specifying log_bin path * mysql: specify a custom server_id * mysql: option to define as read only diff --git a/lxc-php/tasks/php56.yml b/lxc-php/tasks/php56.yml index 1f254b35..b3fb00ac 100644 --- a/lxc-php/tasks/php56.yml +++ b/lxc-php/tasks/php56.yml @@ -3,7 +3,7 @@ - name: "{{ lxc_php_version }} - Install PHP packages" lxc_container: name: "{{ lxc_php_version }}" - container_command: "DEBIAN_FRONTEND=noninteractive apt install -y php5-fpm php5-cli php5-gd php5-imap php5-ldap php5-mcrypt php5-mysql php5-pgsql php-gettext php5-intl php5-curl php5-ssh2 libphp-phpmailer ssmtp" + container_command: "DEBIAN_FRONTEND=noninteractive apt install -y php5-fpm php5-cli php5-gd php5-imap php5-ldap php5-mcrypt php5-mysql php5-pgsql php5-sqlite php-gettext php5-intl php5-curl php5-ssh2 libphp-phpmailer ssmtp" - name: "{{ lxc_php_version }} - Copy evolinux PHP configuration" template: diff --git a/lxc-php/tasks/php70.yml b/lxc-php/tasks/php70.yml index 2e45890d..8cbb0125 100644 --- a/lxc-php/tasks/php70.yml +++ b/lxc-php/tasks/php70.yml @@ -3,7 +3,7 @@ - name: "{{ lxc_php_version }} - Install PHP packages" lxc_container: name: "{{ lxc_php_version }}" - container_command: "DEBIAN_FRONTEND=noninteractive apt install -y php-fpm php-cli php-gd php-intl php-imap php-ldap php-mcrypt php-mysql php-pgsql php-gettext php-curl php-ssh2 php-zip php-mbstring composer libphp-phpmailer" + container_command: "DEBIAN_FRONTEND=noninteractive apt install -y php-fpm php-cli php-gd php-intl php-imap php-ldap php-mcrypt php-mysql php-pgsql php-sqlite3 php-gettext php-curl php-ssh2 php-zip php-mbstring composer libphp-phpmailer" - name: "{{ lxc_php_version }} - Copy evolinux PHP configuration" template: diff --git a/lxc-php/tasks/php73.yml b/lxc-php/tasks/php73.yml index 0149d6ee..eae17e4e 100644 --- a/lxc-php/tasks/php73.yml +++ b/lxc-php/tasks/php73.yml @@ -3,7 +3,7 @@ - name: "{{ lxc_php_version }} - Install PHP packages" lxc_container: name: "{{ lxc_php_version }}" - container_command: "DEBIAN_FRONTEND=noninteractive apt install -y php-fpm php-cli php-gd php-intl php-imap php-ldap php-mysql php-pgsql php-gettext php-curl php-ssh2 php-zip php-mbstring php-zip composer libphp-phpmailer" + container_command: "DEBIAN_FRONTEND=noninteractive apt install -y php-fpm php-cli php-gd php-intl php-imap php-ldap php-mysql php-pgsql php-sqlite3 php-gettext php-curl php-ssh2 php-zip php-mbstring php-zip composer libphp-phpmailer" - name: "{{ lxc_php_version }} - Copy evolinux PHP configuration" template: From 2dcf99a11797aced3c9d9eb60edb20d2aea010aa Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Thu, 4 Jun 2020 11:51:25 +0200 Subject: [PATCH 31/88] lxc-php: Don't disable putenv() by default in PHP settings --- CHANGELOG.md | 1 + lxc-php/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9c40fc8..6e10f240 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The **patch** part changes incrementally at each release. ### Added * lxc-php: Install php-sqlite by default +* lxc-php: Don't disable putenv() by default in PHP settings * mysql: activate binary logs by specifying log_bin path * mysql: specify a custom server_id * mysql: option to define as read only diff --git a/lxc-php/defaults/main.yml b/lxc-php/defaults/main.yml index ef564b2a..8cb62665 100644 --- a/lxc-php/defaults/main.yml +++ b/lxc-php/defaults/main.yml @@ -5,7 +5,7 @@ php_conf_display_errors: "Off" php_conf_log_errors: "On" php_conf_html_errors: "Off" php_conf_allow_url_fopen: "Off" -php_conf_disable_functions: "exec,shell-exec,system,passthru,putenv,popen" +php_conf_disable_functions: "exec,shell-exec,system,passthru,popen" lxc_php_version: Null From c521c98a11290161804fd0ebafb5031e9e087cb7 Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Thu, 4 Jun 2020 11:52:04 +0200 Subject: [PATCH 32/88] php: Don't disable putenv() by default in PHP settings --- CHANGELOG.md | 1 + php/tasks/config_apache.yml | 2 +- php/tasks/config_fpm.yml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e10f240..8d6bcd90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The **patch** part changes incrementally at each release. * mysql: option to define as read only * packweb-apache: Install zip & unzip by default * php: Install php-sqlite by default +* php: Don't disable putenv() by default in PHP settings ### Changed diff --git a/php/tasks/config_apache.yml b/php/tasks/config_apache.yml index 6b4a3200..1ce74733 100644 --- a/php/tasks/config_apache.yml +++ b/php/tasks/config_apache.yml @@ -23,7 +23,7 @@ dest: "{{ php_apache_defaults_ini_file }}" section: PHP option: disable_functions - value: "exec,shell-exec,system,passthru,putenv,popen" + value: "exec,shell-exec,system,passthru,popen" mode: "0644" - name: Custom php.ini diff --git a/php/tasks/config_fpm.yml b/php/tasks/config_fpm.yml index 1f6ae6ed..1644fd6a 100644 --- a/php/tasks/config_fpm.yml +++ b/php/tasks/config_fpm.yml @@ -24,7 +24,7 @@ dest: "{{ php_fpm_defaults_ini_file }}" section: PHP option: disable_functions - value: "exec,shell-exec,system,passthru,putenv,popen" + value: "exec,shell-exec,system,passthru,popen" notify: "restart {{ php_fpm_service_name }}" - name: Custom php.ini for FPM From 5ce715d2ae7911c06316b02e99600cd25af0a0b6 Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Fri, 5 Jun 2020 10:57:49 +0200 Subject: [PATCH 33/88] lxc-php: Install opensmtpd as intended --- CHANGELOG.md | 1 + lxc-php/tasks/mail_opensmtpd.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d6bcd90..6fa25ca3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ The **patch** part changes incrementally at each release. ### Fixed * certbot: restore compatibility with old Nginx +* lxc-php: Install opensmtpd as intended ### Removed diff --git a/lxc-php/tasks/mail_opensmtpd.yml b/lxc-php/tasks/mail_opensmtpd.yml index 20fe91e3..59f6c108 100644 --- a/lxc-php/tasks/mail_opensmtpd.yml +++ b/lxc-php/tasks/mail_opensmtpd.yml @@ -3,7 +3,7 @@ - name: "{{ lxc_php_version }} - Install opensmtpd" lxc_container: name: "{{ lxc_php_version }}" - container_command: "DEBIAN_FRONTEND=noninteractive apt install -y ssmtp" + container_command: "DEBIAN_FRONTEND=noninteractive apt install -y opensmtpd" - name: "{{ lxc_php_version }} - Configure opensmtpd (in the container)" template: From 974bc653f361d763fba696a236be6cfec09ad114 Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Fri, 5 Jun 2020 11:01:22 +0200 Subject: [PATCH 34/88] lxc-php: Do --no-install-recommends for ssmtp/opensmtpd --- CHANGELOG.md | 1 + lxc-php/tasks/mail_opensmtpd.yml | 2 +- lxc-php/tasks/mail_ssmtp.yml | 2 +- lxc-php/tasks/php56.yml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fa25ca3..55334f8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The **patch** part changes incrementally at each release. ### Changed +* lxc-php: Do --no-install-recommends for ssmtp/opensmtpd * packweb-apache: Don't turn on mod-evasive emails by default * haproxy: chroot and socket path are configurable * redis: create sudoers file if missing diff --git a/lxc-php/tasks/mail_opensmtpd.yml b/lxc-php/tasks/mail_opensmtpd.yml index 59f6c108..1b4dbea0 100644 --- a/lxc-php/tasks/mail_opensmtpd.yml +++ b/lxc-php/tasks/mail_opensmtpd.yml @@ -3,7 +3,7 @@ - name: "{{ lxc_php_version }} - Install opensmtpd" lxc_container: name: "{{ lxc_php_version }}" - container_command: "DEBIAN_FRONTEND=noninteractive apt install -y opensmtpd" + container_command: "DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y opensmtpd" - name: "{{ lxc_php_version }} - Configure opensmtpd (in the container)" template: diff --git a/lxc-php/tasks/mail_ssmtp.yml b/lxc-php/tasks/mail_ssmtp.yml index 01e8216a..95055044 100644 --- a/lxc-php/tasks/mail_ssmtp.yml +++ b/lxc-php/tasks/mail_ssmtp.yml @@ -3,7 +3,7 @@ - name: "{{ lxc_php_version }} - Install ssmtp" lxc_container: name: "{{ lxc_php_version }}" - container_command: "DEBIAN_FRONTEND=noninteractive apt install -y ssmtp" + container_command: "DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y ssmtp " - name: "{{ lxc_php_version }} - Configure ssmtp" template: diff --git a/lxc-php/tasks/php56.yml b/lxc-php/tasks/php56.yml index b3fb00ac..b10bb772 100644 --- a/lxc-php/tasks/php56.yml +++ b/lxc-php/tasks/php56.yml @@ -3,7 +3,7 @@ - name: "{{ lxc_php_version }} - Install PHP packages" lxc_container: name: "{{ lxc_php_version }}" - container_command: "DEBIAN_FRONTEND=noninteractive apt install -y php5-fpm php5-cli php5-gd php5-imap php5-ldap php5-mcrypt php5-mysql php5-pgsql php5-sqlite php-gettext php5-intl php5-curl php5-ssh2 libphp-phpmailer ssmtp" + container_command: "DEBIAN_FRONTEND=noninteractive apt install -y php5-fpm php5-cli php5-gd php5-imap php5-ldap php5-mcrypt php5-mysql php5-pgsql php5-sqlite php-gettext php5-intl php5-curl php5-ssh2 libphp-phpmailer" - name: "{{ lxc_php_version }} - Copy evolinux PHP configuration" template: From e380fa5085311a532320c2b8d31c5182a86a2680 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 5 Jun 2020 11:01:32 +0200 Subject: [PATCH 35/88] evoacme: upstream release 20.06.1 --- CHANGELOG.md | 2 +- evoacme/files/evoacme.sh | 2 +- evoacme/files/make-csr.sh | 6 +++--- evoacme/files/vhost-domains.sh | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55334f8b..bc4a85b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ The **patch** part changes incrementally at each release. * redis: create sudoers file if missing * redis: new syntax for match filter * redis: raise an error is port 6379 is used in "instance" mode -* evoacme: upstream release 20.06 +* evoacme: upstream release 20.06.1 * certbot: install certbot dependencies non-interactively for jessie ### Fixed diff --git a/evoacme/files/evoacme.sh b/evoacme/files/evoacme.sh index 3b6d9771..d98f061e 100755 --- a/evoacme/files/evoacme.sh +++ b/evoacme/files/evoacme.sh @@ -303,7 +303,7 @@ readonly QUIET=${QUIET:-"0"} readonly TEST=${TEST:-"0"} readonly DRY_RUN=${DRY_RUN:-"0"} -readonly VERSION="20.06" +readonly VERSION="20.06.1" # Read configuration file, if it exists [ -r /etc/default/evoacme ] && . /etc/default/evoacme diff --git a/evoacme/files/make-csr.sh b/evoacme/files/make-csr.sh index d7abb204..7e08cfe0 100755 --- a/evoacme/files/make-csr.sh +++ b/evoacme/files/make-csr.sh @@ -204,8 +204,8 @@ main() { [ "$1" = "-V" ] || [ "$1" = "--version" ] && show_version && exit 0 if [ -t 0 ]; then - # We have STDIN, so we should have 2 arguments - [ "$#" -eq 2 ] || error "invalid argument(s)" + # We have STDIN, so we should at least 2 arguments + [ "$#" -ge 2 ] || error "invalid argument(s)" # read VHOST from first argument VHOST="$1" @@ -265,7 +265,7 @@ readonly ARGS=$@ readonly VERBOSE=${VERBOSE:-"0"} readonly QUIET=${QUIET:-"0"} -readonly VERSION="20.06" +readonly VERSION="20.06.1" # Read configuration file, if it exists [ -r /etc/default/evoacme ] && . /etc/default/evoacme diff --git a/evoacme/files/vhost-domains.sh b/evoacme/files/vhost-domains.sh index 6bf91286..8b9ef3a5 100755 --- a/evoacme/files/vhost-domains.sh +++ b/evoacme/files/vhost-domains.sh @@ -170,7 +170,7 @@ readonly ARGS=$@ readonly VERBOSE=${VERBOSE:-"0"} readonly QUIET=${QUIET:-"0"} -readonly VERSION="20.06" +readonly VERSION="20.06.1" readonly SRV_IP=${SRV_IP:-""} From 413597cd591f692d4876402e621e45051648149c Mon Sep 17 00:00:00 2001 From: Eric Morino Date: Fri, 5 Jun 2020 10:59:42 +0200 Subject: [PATCH 36/88] Force owner opendkim for /etc/opendkim/ folder --- opendkim/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opendkim/tasks/main.yml b/opendkim/tasks/main.yml index 9b309825..901d03f2 100644 --- a/opendkim/tasks/main.yml +++ b/opendkim/tasks/main.yml @@ -59,6 +59,8 @@ - name: Set folder permissions to 0750 file: path: "/etc/opendkim/" + owner: opendkim + group: opendkim mode: "0750" force: yes tags: From 968d19bb301678863cab04a84c3008f8f975327a Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 5 Jun 2020 11:02:50 +0200 Subject: [PATCH 37/88] mongodb: fix logrotate patterm on Debian buster --- CHANGELOG.md | 2 ++ mongodb/templates/logrotate_buster.j2 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc4a85b3..1e556c71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The **patch** part changes incrementally at each release. ## [Unreleased] ### Added + * lxc-php: Install php-sqlite by default * lxc-php: Don't disable putenv() by default in PHP settings * mysql: activate binary logs by specifying log_bin path @@ -35,6 +36,7 @@ The **patch** part changes incrementally at each release. * certbot: restore compatibility with old Nginx * lxc-php: Install opensmtpd as intended +* mongodb: fix logrotate patterm on Debian buster ### Removed diff --git a/mongodb/templates/logrotate_buster.j2 b/mongodb/templates/logrotate_buster.j2 index fe5926a0..8239e880 100644 --- a/mongodb/templates/logrotate_buster.j2 +++ b/mongodb/templates/logrotate_buster.j2 @@ -1,6 +1,6 @@ # {{ ansible_managed }} -/var/log/mongodb/mongodb.log { +/var/log/mongodb/mongod.log { daily missingok rotate 365 From 99f0d9e17869fdfcc5ed6828301208846b7173a7 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 5 Jun 2020 11:31:42 +0200 Subject: [PATCH 38/88] evoacme: read values from environment before defaults file --- evoacme/templates/evoacme.conf.j2 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/evoacme/templates/evoacme.conf.j2 b/evoacme/templates/evoacme.conf.j2 index 08ead2f1..fc1ba70b 100644 --- a/evoacme/templates/evoacme.conf.j2 +++ b/evoacme/templates/evoacme.conf.j2 @@ -1,8 +1,8 @@ ### File generated by Ansible ### -SSL_KEY_DIR={{ evoacme_ssl_key_dir }} -ACME_DIR={{ evoacme_acme_dir }} -CSR_DIR={{ evoacme_csr_dir }} -CRT_DIR={{ evoacme_crt_dir }} -LOG_DIR={{ evoacme_log_dir }} -SSL_MINDAY={{ evoacme_ssl_minday }} +SSL_KEY_DIR=${SSL_KEY_DIR:-{{ evoacme_ssl_key_dir }}} +ACME_DIR=${ACME_DIR:-{{ evoacme_acme_dir }}} +CSR_DIR=${CSR_DIR:-{{ evoacme_csr_dir }}} +CRT_DIR=${CRT_DIR:-{{ evoacme_crt_dir }}} +LOG_DIR=${LOG_DIR:-{{ evoacme_log_dir }}} +SSL_MINDAY=${SSL_MINDAY:-{{ evoacme_ssl_minday }}} From f5d06ad0b13467988483381ed6b40b63e94bbca3 Mon Sep 17 00:00:00 2001 From: Patrick Marchand Date: Mon, 8 Jun 2020 17:22:18 -0400 Subject: [PATCH 39/88] evobackup-client: Fix ssh connection test in zzz_evobackup.sh When I made the ssh key name a variable and defaulted it to id_ed25519, I forgot to change the hardcoded value for the ssh test in evobackup-client/templates/zzz_evobackup.default.sh.j2 --- CHANGELOG.md | 1 + evobackup-client/templates/zzz_evobackup.default.sh.j2 | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e556c71..dc08a807 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ The **patch** part changes incrementally at each release. * certbot: restore compatibility with old Nginx * lxc-php: Install opensmtpd as intended * mongodb: fix logrotate patterm on Debian buster +* evobackup-client: fixed the ssh connection test ### Removed diff --git a/evobackup-client/templates/zzz_evobackup.default.sh.j2 b/evobackup-client/templates/zzz_evobackup.default.sh.j2 index f3674413..49de9744 100644 --- a/evobackup-client/templates/zzz_evobackup.default.sh.j2 +++ b/evobackup-client/templates/zzz_evobackup.default.sh.j2 @@ -1,5 +1,5 @@ #!/bin/sh -# Careful, the zzz_evobackup template was last updated on 2020/04/15 +# Careful, the zzz_evobackup template was last updated on 2020/06/08 # # Script Evobackup client # See https://gitea.evolix.org/evolix/evobackup @@ -76,7 +76,7 @@ test_server() { port=$(echo "${item}" | cut -d':' -f2) # Test if the server is accepting connections - ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" -i /root/.ssh/evobackup_id "${host}" -p "${port}" -t "exit" + ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" -i {{ evobackup_client__root_key_path }} "${host}" -p "${port}" -t "exit" # shellcheck disable=SC2181 if [ $? = 0 ]; then # SSH connection is OK From 45731c77554b3d928132c692001a7e3f256d8191 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 9 Jun 2020 11:41:26 +0200 Subject: [PATCH 40/88] haproxy: enable stats frontend with access lists --- CHANGELOG.md | 1 + haproxy/defaults/main.yml | 7 +++++ haproxy/tasks/main.yml | 36 ++++++++++++++++++++++++ haproxy/templates/haproxy.default.cfg.j2 | 20 +++++++++++++ 4 files changed, 64 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc08a807..bd0dd90f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The **patch** part changes incrementally at each release. ### Added +* haproxy: enable stats frontend with access lists * lxc-php: Install php-sqlite by default * lxc-php: Don't disable putenv() by default in PHP settings * mysql: activate binary logs by specifying log_bin path diff --git a/haproxy/defaults/main.yml b/haproxy/defaults/main.yml index a0f4f259..6096567a 100644 --- a/haproxy/defaults/main.yml +++ b/haproxy/defaults/main.yml @@ -7,3 +7,10 @@ haproxy_force_config: True haproxy_socket: /run/haproxy/admin.sock haproxy_chroot: /var/lib/haproxy + +haproxy_stats_access_ips: [] +haproxy_stats_admin_ips: [] +haproxy_maintenance_ips: [] + +haproxy_stats_enable: False +haproxy_stats_bind: "*:8080 ssl crt /etc/haproxy/ssl/" diff --git a/haproxy/tasks/main.yml b/haproxy/tasks/main.yml index 8e3094fb..f17d14bc 100644 --- a/haproxy/tasks/main.yml +++ b/haproxy/tasks/main.yml @@ -35,4 +35,40 @@ - haproxy - config +- name: HAProxy stats_access_ips are present + blockinfile: + dest: /etc/haproxy/stats_access_ips + create: yes + block: | + {% for ip in haproxy_stats_access_ips | default([]) %} + {{ ip }} + {% endfor %} + notify: reload haproxy + tags: + - haproxy + - config + +- name: HAProxy stats_admin_ips are present + blockinfile: + dest: /etc/haproxy/stats_admin_ips + create: yes + block: | + {% for ip in haproxy_stats_admin_ips | default([]) %} + {{ ip }} + {% endfor %} + notify: reload haproxy + tags: + - haproxy + - config + +- name: HAProxy maintenance_ips are present + blockinfile: + dest: /etc/haproxy/maintenance_ips + create: yes + block: | + {% for ip in haproxy_maintenance_ips | default([]) %} + {{ ip }} + {% endfor %} + notify: reload haproxy + - include: munin.yml diff --git a/haproxy/templates/haproxy.default.cfg.j2 b/haproxy/templates/haproxy.default.cfg.j2 index 583cb347..09e0e3ff 100644 --- a/haproxy/templates/haproxy.default.cfg.j2 +++ b/haproxy/templates/haproxy.default.cfg.j2 @@ -37,3 +37,23 @@ defaults errorfile 502 /etc/haproxy/errors/502.http errorfile 503 /etc/haproxy/errors/503.http errorfile 504 /etc/haproxy/errors/504.http + +{% if haproxy_stats_enable %} +listen stats + mode http + bind {{ haproxy_stats_bind }} + + stats enable + stats refresh 10s + stats uri / + stats show-legends + stats show-node + + acl stats_access_ips src -f /etc/haproxy/stats_access_ips + http-request deny if !stats_access_ips + + acl stats_admin_ips src -f /etc/haproxy/stats_admin_ips + stats admin if stats_admin_ips + + http-request set-log-level silent +{% endif %} From 7f65505f018a5e110e94da2ac12d900159b71469 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 9 Jun 2020 11:45:19 +0200 Subject: [PATCH 41/88] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd0dd90f..6f4e3091 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The **patch** part changes incrementally at each release. * redis: new syntax for match filter * redis: raise an error is port 6379 is used in "instance" mode * evoacme: upstream release 20.06.1 +* evoacme: read values from environment before defaults file * certbot: install certbot dependencies non-interactively for jessie ### Fixed From 9bfbbf08c4ae44e9a7ac6807cfccb8a69c75d97d Mon Sep 17 00:00:00 2001 From: Patrick Marchand Date: Wed, 10 Jun 2020 10:37:36 -0400 Subject: [PATCH 42/88] =?UTF-8?q?Fix=20variable=20par=20d=C3=A9faut=20manq?= =?UTF-8?q?uante=20dans=20mysql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Le lower_case_table_names doit être mis à Null pour que le check marche. --- mysql/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysql/defaults/main.yml b/mysql/defaults/main.yml index 8d8771fa..1e2f673d 100644 --- a/mysql/defaults/main.yml +++ b/mysql/defaults/main.yml @@ -31,6 +31,7 @@ mysql_query_cache_size: Null mysql_max_allowed_packet: Null mysql_force_custom_config: 'no' mysql_innodb_log_file_size: Null +mysql_lower_case_table_names: Null mysql_cron_optimize: True mysql_cron_optimize_frequency: weekly @@ -53,3 +54,4 @@ mysql_server_id: null mysql_bind_address: null mysql_repl_password: '' mysql_read_only: 0 + From 39b8e9698d1d9239ca71fc874a14f6a2274e29d9 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 14 Jun 2020 12:30:34 +0200 Subject: [PATCH 43/88] certbot: detect HAProxy cert directory --- CHANGELOG.md | 1 + certbot/files/hooks/haproxy.sh | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f4e3091..eb83f6aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The **patch** part changes incrementally at each release. ### Added +* certbot: detect HAProxy cert directory * haproxy: enable stats frontend with access lists * lxc-php: Install php-sqlite by default * lxc-php: Don't disable putenv() by default in PHP settings diff --git a/certbot/files/hooks/haproxy.sh b/certbot/files/hooks/haproxy.sh index 8bb66e2c..4998d55f 100644 --- a/certbot/files/hooks/haproxy.sh +++ b/certbot/files/hooks/haproxy.sh @@ -16,7 +16,7 @@ found_renewed_lineage() { test -f "${RENEWED_LINEAGE}/fullchain.pem" && test -f "${RENEWED_LINEAGE}/privkey.pem" } config_check() { - ${haproxy_bin} -c -f /etc/haproxy/haproxy.cfg > /dev/null 2>&1 + ${haproxy_bin} -c -f "${haproxy_config_file}" > /dev/null 2>&1 } concat_files() { # shellcheck disable=SC2174 @@ -34,6 +34,22 @@ cert_and_key_mismatch() { test "${haproxy_cert_md5}" != "${haproxy_key_md5}" } +detect_haproxy_cert_dir() { + # get last field or line wich defines the crt directory + config_cert_dir=$(grep -r -o -E -h '^\s*bind .* crt /etc/.+\b' "${haproxy_config_file}" | head -1 | awk '{ print $(NF)}') + if [ -n "${config_cert_dir}" ]; then + debug "Cert directory is configured with ${config_cert_dir}" + echo "${config_cert_dir}" + elif [ -d "/etc/haproxy/ssl" ]; then + debug "No configured cert directory found, but /etc/haproxy/ssl exists" + echo "/etc/haproxy/ssl" + elif [ -d "/etc/ssl/haproxy" ]; then + debug "No configured cert directory found, but /etc/ssl/haproxy exists" + echo "/etc/ssl/haproxy" + else + error "Cert directory not found." + fi +} main() { if [ -z "${RENEWED_LINEAGE}" ]; then error "This script must be called only by certbot!" @@ -70,6 +86,7 @@ readonly VERBOSE=${VERBOSE:-"0"} readonly QUIET=${QUIET:-"0"} readonly haproxy_bin=$(command -v haproxy) -readonly haproxy_cert_dir="/etc/ssl/haproxy" +readonly haproxy_config_file="/etc/haproxy/haproxy.cfg" +readonly haproxy_cert_dir=$(detect_haproxy_cert_dir) main From d5b29a2fe09ac0c7f6c1d56ccdd7540cdae35011 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 14 Jun 2020 12:32:14 +0200 Subject: [PATCH 44/88] nginx: make default vhost configurable --- CHANGELOG.md | 1 + nginx/defaults/main.yml | 7 +++++++ nginx/tasks/main_minimal.yml | 3 ++- nginx/tasks/main_regular.yml | 4 ++-- nginx/tasks/packages.yml | 10 +++++++++- 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb83f6aa..e5574db4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The **patch** part changes incrementally at each release. * mysql: activate binary logs by specifying log_bin path * mysql: specify a custom server_id * mysql: option to define as read only +* nginx: make default vhost configurable * packweb-apache: Install zip & unzip by default * php: Install php-sqlite by default * php: Don't disable putenv() by default in PHP settings diff --git a/nginx/defaults/main.yml b/nginx/defaults/main.yml index 932356b0..d59da758 100644 --- a/nginx/defaults/main.yml +++ b/nginx/defaults/main.yml @@ -22,3 +22,10 @@ nginx_evolinux_default_enabled: True nginx_serverstatus_suffix: "" nginx_serverstatus_suffix_file: "/etc/evolinux/nginx_serverstatus_suffix" + +nginx_force_default_template: False +nginx_default_template_regular: "evolinux-default.conf.j2" +nginx_default_template_minimal: "evolinux-default.minimal.conf.j2" + +nginx_service_state: started +nginx_service_enabled: True diff --git a/nginx/tasks/main_minimal.yml b/nginx/tasks/main_minimal.yml index 58d177af..798cf055 100644 --- a/nginx/tasks/main_minimal.yml +++ b/nginx/tasks/main_minimal.yml @@ -12,9 +12,10 @@ - name: Copy default vhost template: - src: evolinux-default.minimal.conf.j2 + src: "{{ nginx_default_template_minimal }}" dest: /etc/nginx/sites-available/evolinux-default.minimal.conf mode: 0644 + force: "{{ nginx_force_default_template | default(False) }}" notify: reload nginx tags: - nginx diff --git a/nginx/tasks/main_regular.yml b/nginx/tasks/main_regular.yml index 99080a5e..45e5ba47 100644 --- a/nginx/tasks/main_regular.yml +++ b/nginx/tasks/main_regular.yml @@ -89,10 +89,10 @@ - name: nginx vhost is installed template: - src: evolinux-default.conf.j2 + src: "{{ nginx_default_template_regular }}" dest: /etc/nginx/sites-available/evolinux-default.conf mode: "0640" - force: no + force: "{{ nginx_force_default_template | default(False) }}" notify: reload nginx tags: - nginx diff --git a/nginx/tasks/packages.yml b/nginx/tasks/packages.yml index ccbd3705..76350424 100644 --- a/nginx/tasks/packages.yml +++ b/nginx/tasks/packages.yml @@ -7,7 +7,15 @@ apt: name: "{{ nginx_package_name }}" state: present - notify: restart nginx + tags: + - nginx + - packages + +- name: Ensure nginx service is running as configured. + service: + name: nginx + state: "{{ nginx_service_state }}" + enabled: "{{ nginx_service_enabled }}" tags: - nginx - packages From e197f5c2ba4f0bafd60529ad1979b78f9d484b65 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 14 Jun 2020 12:35:44 +0200 Subject: [PATCH 45/88] haproxy: adapt backports installed package list to distibution --- CHANGELOG.md | 1 + haproxy/defaults/main.yml | 3 +++ haproxy/tasks/packages_backports.yml | 20 ++++++++++++++------ haproxy/templates/haproxy_apt_preferences.j2 | 2 +- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5574db4..ae6a2d2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ The **patch** part changes incrementally at each release. * lxc-php: Do --no-install-recommends for ssmtp/opensmtpd * packweb-apache: Don't turn on mod-evasive emails by default * haproxy: chroot and socket path are configurable +* haproxy: adapt backports installed package list to distibution * redis: create sudoers file if missing * redis: new syntax for match filter * redis: raise an error is port 6379 is used in "instance" mode diff --git a/haproxy/defaults/main.yml b/haproxy/defaults/main.yml index 6096567a..64d061ef 100644 --- a/haproxy/defaults/main.yml +++ b/haproxy/defaults/main.yml @@ -14,3 +14,6 @@ haproxy_maintenance_ips: [] haproxy_stats_enable: False haproxy_stats_bind: "*:8080 ssl crt /etc/haproxy/ssl/" + +haproxy_backports_packages_stretch: haproxy libssl1.0.0 +haproxy_backports_packages_buster: haproxy diff --git a/haproxy/tasks/packages_backports.yml b/haproxy/tasks/packages_backports.yml index 66c2118e..9a682120 100644 --- a/haproxy/tasks/packages_backports.yml +++ b/haproxy/tasks/packages_backports.yml @@ -4,8 +4,16 @@ name: evolix/apt tasks_from: backports.yml tags: - - haproxy - - packages + - haproxy + - packages + +- set_fact: + haproxy_backports_packages: "{{ haproxy_backports_packages_stretch }}" + when: ansible_distribution_release == 'stretch' + +- set_fact: + haproxy_backports_packages: "{{ haproxy_backports_packages_buster }}" + when: ansible_distribution_release == 'buster' - name: Prefer HAProxy package from backports template: @@ -15,13 +23,13 @@ mode: "0640" register: haproxy_apt_preferences tags: - - haproxy - - packages + - haproxy + - packages - name: update apt apt: update_cache: yes when: haproxy_apt_preferences is changed tags: - - haproxy - - packages + - haproxy + - packages diff --git a/haproxy/templates/haproxy_apt_preferences.j2 b/haproxy/templates/haproxy_apt_preferences.j2 index 706ee67d..243e3e86 100644 --- a/haproxy/templates/haproxy_apt_preferences.j2 +++ b/haproxy/templates/haproxy_apt_preferences.j2 @@ -1,3 +1,3 @@ -Package: haproxy libssl1.0.0 +Package: {{ haproxy_backports_packages }} Pin: release a={{ ansible_distribution_release }}-backports Pin-Priority: 999 From a381d23d1f908e380840b8abb56bfd521a7adc01 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 14 Jun 2020 12:36:58 +0200 Subject: [PATCH 46/88] haproxy: preconfigure SSL with defaults --- CHANGELOG.md | 1 + haproxy/tasks/main.yml | 73 +++++++++++++++++------- haproxy/templates/haproxy.default.cfg.j2 | 13 +++-- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae6a2d2d..5277e978 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The **patch** part changes incrementally at each release. * certbot: detect HAProxy cert directory * haproxy: enable stats frontend with access lists +* haproxy: preconfigure SSL with defaults * lxc-php: Install php-sqlite by default * lxc-php: Don't disable putenv() by default in PHP settings * mysql: activate binary logs by specifying log_bin path diff --git a/haproxy/tasks/main.yml b/haproxy/tasks/main.yml index f17d14bc..9fade1a6 100644 --- a/haproxy/tasks/main.yml +++ b/haproxy/tasks/main.yml @@ -7,30 +7,35 @@ - haproxy - packages -- include: packages_backports.yml - when: haproxy_backports - -- name: Install HAProxy package - apt: - name: haproxy - state: present +- name: HAProxy SSL directory is present + file: + path: /etc/haproxy/ssl + owner: root + group: root + mode: "0700" + state: directory tags: - haproxy - - packages + - config -- name: Copy HAProxy configuration - template: - src: "{{ item }}" - dest: /etc/haproxy/haproxy.cfg - force: "{{ haproxy_force_config }}" - validate: "haproxy -c -f %s" - with_first_found: - - "templates/haproxy/haproxy.{{ inventory_hostname }}.cfg.j2" - - "templates/haproxy/haproxy.{{ host_group }}.cfg.j2" - - "templates/haproxy/haproxy.default.cfg.j2" - - "haproxy.default.cfg.j2" +- name: Self-signed certificate is present in HAProxy ssl directory + shell: "cat /etc/ssl/certs/ssl-cert-snakeoil.pem /etc/ssl/private/ssl-cert-snakeoil.key > /etc/haproxy/ssl/ssl-cert-snakeoil.pem" + args: + creates: /etc/haproxy/ssl/ssl-cert-snakeoil.pem + notify: reload haproxy + tags: + - haproxy + - config + +- name: 2048 bits DHparam file is present + get_url: + url: https://ssl-config.mozilla.org/ffdhe2048.txt + dest: /etc/haproxy/dhparam2048.txt + mode: '0600' + owner: root + group: root + force: no notify: reload haproxy - when: haproxy_update_config tags: - haproxy - config @@ -71,4 +76,32 @@ {% endfor %} notify: reload haproxy +- include: packages_backports.yml + when: haproxy_backports + +- name: Install HAProxy package + apt: + name: haproxy + state: present + tags: + - haproxy + - packages + +- name: Copy HAProxy configuration + template: + src: "{{ item }}" + dest: /etc/haproxy/haproxy.cfg + force: "{{ haproxy_force_config }}" + validate: "haproxy -c -f %s" + with_first_found: + - "templates/haproxy/haproxy.{{ inventory_hostname }}.cfg.j2" + - "templates/haproxy/haproxy.{{ host_group }}.cfg.j2" + - "templates/haproxy/haproxy.default.cfg.j2" + - "haproxy.default.cfg.j2" + notify: reload haproxy + when: haproxy_update_config + tags: + - haproxy + - config + - include: munin.yml diff --git a/haproxy/templates/haproxy.default.cfg.j2 b/haproxy/templates/haproxy.default.cfg.j2 index 09e0e3ff..b3a290eb 100644 --- a/haproxy/templates/haproxy.default.cfg.j2 +++ b/haproxy/templates/haproxy.default.cfg.j2 @@ -14,11 +14,14 @@ global ca-base /etc/ssl/certs crt-base /etc/ssl/private - # Default ciphers to use on SSL-enabled listening sockets. - # For more information, see ciphers(1SSL). This list is from: - # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ - ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS - ssl-default-bind-options no-sslv3 + # intermediate configuration https://ssl-config.mozilla.org/ + ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 + ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets + + ssl-default-server-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 + ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets + + ssl-dh-param-file /etc/haproxy/dhparam2048.txt defaults log global From c2ae3de929e6398f95e98dd09147a21bd49f0ff1 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 14 Jun 2020 12:49:10 +0200 Subject: [PATCH 47/88] nginx: read server-status values before changing the config --- CHANGELOG.md | 1 + nginx/tasks/main_regular.yml | 6 +++++- ...rver_status.yml => server_status_read.yml} | 19 ------------------ nginx/tasks/server_status_write.yml | 20 +++++++++++++++++++ 4 files changed, 26 insertions(+), 20 deletions(-) rename nginx/tasks/{server_status.yml => server_status_read.yml} (62%) create mode 100644 nginx/tasks/server_status_write.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 5277e978..353d5280 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The **patch** part changes incrementally at each release. * packweb-apache: Don't turn on mod-evasive emails by default * haproxy: chroot and socket path are configurable * haproxy: adapt backports installed package list to distibution +* nginx: read server-status values before changing the config * redis: create sudoers file if missing * redis: new syntax for match filter * redis: raise an error is port 6379 is used in "instance" mode diff --git a/nginx/tasks/main_regular.yml b/nginx/tasks/main_regular.yml index 45e5ba47..c7989bee 100644 --- a/nginx/tasks/main_regular.yml +++ b/nginx/tasks/main_regular.yml @@ -2,6 +2,10 @@ - include: packages.yml +- include: server_status_read.yml + tags: + - nginx + # TODO: find a way to override the main configuration # without touching the main file @@ -108,7 +112,7 @@ tags: - nginx -- include: server_status.yml +- include: server_status_write.yml tags: - nginx diff --git a/nginx/tasks/server_status.yml b/nginx/tasks/server_status_read.yml similarity index 62% rename from nginx/tasks/server_status.yml rename to nginx/tasks/server_status_read.yml index 14bb3fe5..570febf9 100644 --- a/nginx/tasks/server_status.yml +++ b/nginx/tasks/server_status_read.yml @@ -34,22 +34,3 @@ - debug: var: nginx_serverstatus_suffix verbosity: 1 - -- name: replace server-status suffix in default site index - replace: - dest: /var/www/index.html - regexp: '__SERVERSTATUS_SUFFIX__' - replace: "{{ nginx_serverstatus_suffix }}" - -- name: add server-status suffix in default site index if missing - replace: - dest: /var/www/index.html - regexp: '"/nginx_status-?"' - replace: '"/nginx_status-{{ nginx_serverstatus_suffix }}"' - -- name: add server-status suffix in default VHost - replace: - dest: /etc/nginx/sites-available/evolinux-default.conf - regexp: 'location /nginx_status-? {' - replace: 'location /nginx_status-{{ nginx_serverstatus_suffix }} {' - notify: reload nginx diff --git a/nginx/tasks/server_status_write.yml b/nginx/tasks/server_status_write.yml new file mode 100644 index 00000000..e0b069db --- /dev/null +++ b/nginx/tasks/server_status_write.yml @@ -0,0 +1,20 @@ +--- + +- name: replace server-status suffix in default site index + replace: + dest: /var/www/index.html + regexp: '__SERVERSTATUS_SUFFIX__' + replace: "{{ nginx_serverstatus_suffix }}" + +- name: add server-status suffix in default site index if missing + replace: + dest: /var/www/index.html + regexp: '"/nginx_status-?"' + replace: '"/nginx_status-{{ nginx_serverstatus_suffix }}"' + +- name: add server-status suffix in default VHost + replace: + dest: /etc/nginx/sites-available/evolinux-default.conf + regexp: 'location /nginx_status-? {' + replace: 'location /nginx_status-{{ nginx_serverstatus_suffix }} {' + notify: reload nginx From d9f8c4fc897801f834347b0cbcdd2092889ce45b Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 14 Jun 2020 19:47:16 +0200 Subject: [PATCH 48/88] whitespaces --- varnish/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varnish/defaults/main.yml b/varnish/defaults/main.yml index 0cd95b41..544d0cf7 100644 --- a/varnish/defaults/main.yml +++ b/varnish/defaults/main.yml @@ -1,7 +1,7 @@ --- varnish_addresses: -- 0.0.0.0:80 + - 0.0.0.0:80 varnish_management_address: localhost:6082 From 17b49ccc3dd509a0d757b993f7ce0cad43f1ff2d Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 14 Jun 2020 23:19:40 +0200 Subject: [PATCH 49/88] haproxy: simplify syntax fos stats section --- haproxy/templates/haproxy.default.cfg.j2 | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/haproxy/templates/haproxy.default.cfg.j2 b/haproxy/templates/haproxy.default.cfg.j2 index b3a290eb..fb63e1f6 100644 --- a/haproxy/templates/haproxy.default.cfg.j2 +++ b/haproxy/templates/haproxy.default.cfg.j2 @@ -51,12 +51,8 @@ listen stats stats uri / stats show-legends stats show-node + stats admin if { src -f /etc/haproxy/stats_admin_ips } - acl stats_access_ips src -f /etc/haproxy/stats_access_ips - http-request deny if !stats_access_ips - - acl stats_admin_ips src -f /etc/haproxy/stats_admin_ips - stats admin if stats_admin_ips - + http-request deny if !{ src -f /etc/haproxy/stats_access_ips } http-request set-log-level silent {% endif %} From d26d501b2c96f072c94252bbfedf75ee358bbf7e Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 14 Jun 2020 23:27:29 +0200 Subject: [PATCH 50/88] haproxy: add some comments to default config --- CHANGELOG.md | 1 + haproxy/templates/haproxy.default.cfg.j2 | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 353d5280..360945fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The **patch** part changes incrementally at each release. ### Added * certbot: detect HAProxy cert directory +* haproxy: add some comments to default config * haproxy: enable stats frontend with access lists * haproxy: preconfigure SSL with defaults * lxc-php: Install php-sqlite by default diff --git a/haproxy/templates/haproxy.default.cfg.j2 b/haproxy/templates/haproxy.default.cfg.j2 index fb63e1f6..43576ce9 100644 --- a/haproxy/templates/haproxy.default.cfg.j2 +++ b/haproxy/templates/haproxy.default.cfg.j2 @@ -56,3 +56,27 @@ listen stats http-request deny if !{ src -f /etc/haproxy/stats_access_ips } http-request set-log-level silent {% endif %} + +# frontend http-https +# bind 0.0.0.0:80 +# bind 0.0.0.0:443 ssl crt /etc/haproxy/ssl +# +# capture request header Host len 32 +# +# option forwardfor +# +# acl self hdr(host) -i {{ ansible_fqdn }} +# +# # Detect Let's Encrypt challenge requests +# acl letsencrypt path_dir -i /.well-known/acme-challenge +# +# http-request set-header X-Forwarded-Proto https if { ssl_fc } +# http-request set-header X-Forwarded-Port 443 if { ssl_fc } +# +# use_backend local if letsencrypt || self +# +# backend local +# mode http +# option forwardfor +# +# server localhost 127.0.0.1:81 send-proxy-v2 From 4c4771b3ab783543b54dd1ec092a2a4763696886 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 14 Jun 2020 23:28:29 +0200 Subject: [PATCH 51/88] haproxy: add deny_ips file to reject connections --- CHANGELOG.md | 1 + haproxy/defaults/main.yml | 1 + haproxy/tasks/main.yml | 10 ++++++++++ haproxy/templates/haproxy.default.cfg.j2 | 3 +++ 4 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 360945fc..5e153fdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The **patch** part changes incrementally at each release. ### Added * certbot: detect HAProxy cert directory +* haproxy: add deny_ips file to reject connections * haproxy: add some comments to default config * haproxy: enable stats frontend with access lists * haproxy: preconfigure SSL with defaults diff --git a/haproxy/defaults/main.yml b/haproxy/defaults/main.yml index 64d061ef..55452db8 100644 --- a/haproxy/defaults/main.yml +++ b/haproxy/defaults/main.yml @@ -11,6 +11,7 @@ haproxy_chroot: /var/lib/haproxy haproxy_stats_access_ips: [] haproxy_stats_admin_ips: [] haproxy_maintenance_ips: [] +haproxy_deny_ips: [] haproxy_stats_enable: False haproxy_stats_bind: "*:8080 ssl crt /etc/haproxy/ssl/" diff --git a/haproxy/tasks/main.yml b/haproxy/tasks/main.yml index 9fade1a6..f4ce3d79 100644 --- a/haproxy/tasks/main.yml +++ b/haproxy/tasks/main.yml @@ -76,6 +76,16 @@ {% endfor %} notify: reload haproxy +- name: HAProxy deny_ips are present + blockinfile: + dest: /etc/haproxy/deny_ips + create: yes + block: | + {% for ip in haproxy_deny_ips | default([]) %} + {{ ip }} + {% endfor %} + notify: reload haproxy + - include: packages_backports.yml when: haproxy_backports diff --git a/haproxy/templates/haproxy.default.cfg.j2 b/haproxy/templates/haproxy.default.cfg.j2 index 43576ce9..35f8a858 100644 --- a/haproxy/templates/haproxy.default.cfg.j2 +++ b/haproxy/templates/haproxy.default.cfg.j2 @@ -70,6 +70,9 @@ listen stats # # Detect Let's Encrypt challenge requests # acl letsencrypt path_dir -i /.well-known/acme-challenge # +# # Reject the request at the TCP level if source is in the denylist +# tcp-request connection reject if { src -f /etc/haproxy/deny_ips } +# # http-request set-header X-Forwarded-Proto https if { ssl_fc } # http-request set-header X-Forwarded-Port 443 if { ssl_fc } # From 2a5195078cbd8b332b27e406e2d5eb2f594310a8 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 15 Jun 2020 22:45:22 +0200 Subject: [PATCH 52/88] haproxy: split stats variables --- CHANGELOG.md | 1 + haproxy/defaults/main.yml | 12 +++++++++++- haproxy/templates/haproxy.default.cfg.j2 | 6 +++--- haproxy/templates/munin.conf.j2 | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e153fdd..7e83b8b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ The **patch** part changes incrementally at each release. * packweb-apache: Don't turn on mod-evasive emails by default * haproxy: chroot and socket path are configurable * haproxy: adapt backports installed package list to distibution +* haproxy: split stats variables * nginx: read server-status values before changing the config * redis: create sudoers file if missing * redis: new syntax for match filter diff --git a/haproxy/defaults/main.yml b/haproxy/defaults/main.yml index 55452db8..42f5483b 100644 --- a/haproxy/defaults/main.yml +++ b/haproxy/defaults/main.yml @@ -1,7 +1,15 @@ --- # backward compatibility with a previously used variable +haproxy_stats_ssl: True +haproxy_stats_host: "*" +haproxy_stats_port: "8080" +haproxy_stats_path: "/" +haproxy_stats_bind_directive: "{{ haproxy_stats_host }}:{{ haproxy_stats_port }} {% if haproxy_stats_ssl %}ssl crt {{ haproxy_ssl_dir }}{% endif %}" + +haproxy_stats_internal_url: "{% if haproxy_stats_ssl %}https:{% else %}http:{% endif %}//{% if haproxy_stats_host == '*' or haproxy_stats_host == '0.0.0.0' %}127.0.0.1{% else %}{{ haproxy_stats_host }}{% endif %}:{{ haproxy_stats_port }}{{ haproxy_stats_path }}" +haproxy_stats_external_url: "{% if haproxy_stats_ssl %}https:{% else %}http:{% endif %}//{{ ansible_fqdn }}:{{ haproxy_stats_port }}{{ haproxy_stats_path }}" + haproxy_backports: "{{ haproxy_jessie_backports | default(false, true) }}" -haproxy_stats_url: "http://127.0.0.1:8080/" haproxy_update_config: True haproxy_force_config: True @@ -13,6 +21,8 @@ haproxy_stats_admin_ips: [] haproxy_maintenance_ips: [] haproxy_deny_ips: [] +haproxy_ssl_dir: "/etc/haproxy/ssl/" + haproxy_stats_enable: False haproxy_stats_bind: "*:8080 ssl crt /etc/haproxy/ssl/" diff --git a/haproxy/templates/haproxy.default.cfg.j2 b/haproxy/templates/haproxy.default.cfg.j2 index 35f8a858..5f1f4ed5 100644 --- a/haproxy/templates/haproxy.default.cfg.j2 +++ b/haproxy/templates/haproxy.default.cfg.j2 @@ -44,11 +44,11 @@ defaults {% if haproxy_stats_enable %} listen stats mode http - bind {{ haproxy_stats_bind }} + bind {{ haproxy_stats_bind_directive }} stats enable stats refresh 10s - stats uri / + stats uri {{ haproxy_stats_path }} stats show-legends stats show-node stats admin if { src -f /etc/haproxy/stats_admin_ips } @@ -59,7 +59,7 @@ listen stats # frontend http-https # bind 0.0.0.0:80 -# bind 0.0.0.0:443 ssl crt /etc/haproxy/ssl +# bind 0.0.0.0:443 ssl crt {{ haproxy_ssl_dir }} # # capture request header Host len 32 # diff --git a/haproxy/templates/munin.conf.j2 b/haproxy/templates/munin.conf.j2 index c7500521..7a423600 100644 --- a/haproxy/templates/munin.conf.j2 +++ b/haproxy/templates/munin.conf.j2 @@ -1,2 +1,2 @@ [haproxy_*] -env.url {{ haproxy_stats_url }};csv;norefresh +env.url {{ haproxy_stats_internal_url }};csv;norefresh From d67be3cd91d5a82f4fc4e4a8c659bc47acd7a29c Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 15 Jun 2020 22:47:08 +0200 Subject: [PATCH 53/88] haproxy: deport SSL tuning to Mozilla SSL generator There are too many combinations and they change every so often. It's better to direct the user to the generator to have a good configuration. --- CHANGELOG.md | 1 + haproxy/tasks/main.yml | 13 ------------- haproxy/templates/haproxy.default.cfg.j2 | 9 +-------- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e83b8b9..4ef521e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The **patch** part changes incrementally at each release. * lxc-php: Do --no-install-recommends for ssmtp/opensmtpd * packweb-apache: Don't turn on mod-evasive emails by default +* haproxy: deport SSL tuning to Mozilla SSL generator * haproxy: chroot and socket path are configurable * haproxy: adapt backports installed package list to distibution * haproxy: split stats variables diff --git a/haproxy/tasks/main.yml b/haproxy/tasks/main.yml index f4ce3d79..04f08d39 100644 --- a/haproxy/tasks/main.yml +++ b/haproxy/tasks/main.yml @@ -27,19 +27,6 @@ - haproxy - config -- name: 2048 bits DHparam file is present - get_url: - url: https://ssl-config.mozilla.org/ffdhe2048.txt - dest: /etc/haproxy/dhparam2048.txt - mode: '0600' - owner: root - group: root - force: no - notify: reload haproxy - tags: - - haproxy - - config - - name: HAProxy stats_access_ips are present blockinfile: dest: /etc/haproxy/stats_access_ips diff --git a/haproxy/templates/haproxy.default.cfg.j2 b/haproxy/templates/haproxy.default.cfg.j2 index 5f1f4ed5..5e4c4b9f 100644 --- a/haproxy/templates/haproxy.default.cfg.j2 +++ b/haproxy/templates/haproxy.default.cfg.j2 @@ -14,14 +14,7 @@ global ca-base /etc/ssl/certs crt-base /etc/ssl/private - # intermediate configuration https://ssl-config.mozilla.org/ - ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 - ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets - - ssl-default-server-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 - ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets - - ssl-dh-param-file /etc/haproxy/dhparam2048.txt + # Go to https://ssl-config.mozilla.org/ and build your SSL configuration defaults log global From 39c8daacf4c175df7a4c2fbb0565fbad02da8d37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Tue, 16 Jun 2020 10:36:24 +0900 Subject: [PATCH 54/88] nagios-nrpe: check_hpraid: Be sure that variables are bound --- nagios-nrpe/files/plugins/check_hpraid | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nagios-nrpe/files/plugins/check_hpraid b/nagios-nrpe/files/plugins/check_hpraid index 2dc567b4..42911dd3 100644 --- a/nagios-nrpe/files/plugins/check_hpraid +++ b/nagios-nrpe/files/plugins/check_hpraid @@ -19,6 +19,13 @@ set -euo pipefail # using HPE Smart Storage Administrator. It should support Debian 9 and over. # (Array, controller, cache, battery, etc...) # +# Known working RAID controllers: +# +# - Adaptec Smart Storage PQI 12G SAS/PCIe 3 (rev 01) +# | Smart Array P408i-a SR Gen10 +# | Smart Array E208i-a SR Gen10 +# +# # NOTE: # # You need to install the proprietary tool HPE Smart Storage Administrator (ssacli) from: @@ -241,6 +248,9 @@ if [ "$EXCLUDE_BATTERY" = "1" ]; then check=$(grep -v 'Cache Status: Temporarily Disabled' "$check") fi +check=${check:-} +check2=${check2:-} +check3=${check3:-} if grep -qiE Failed <<< "$check"; then echo "RAID CRITICAL - HP Smart Array Failed: ${check}" exit "$STATE_CRITICAL" From 2a065abf5fb7e0872a7f4292bc5228e238b53f79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Tue, 16 Jun 2020 10:53:00 +0900 Subject: [PATCH 55/88] nagios-nrpe: check_hpraid: Fix wrong command name in examples --- nagios-nrpe/files/plugins/check_hpraid | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nagios-nrpe/files/plugins/check_hpraid b/nagios-nrpe/files/plugins/check_hpraid index 42911dd3..5f140059 100644 --- a/nagios-nrpe/files/plugins/check_hpraid +++ b/nagios-nrpe/files/plugins/check_hpraid @@ -38,11 +38,11 @@ set -euo pipefail # # Examples: # -# ./check_cciss +# ./check_hpraid # ---------------- # RAID OK # -# ./check_cciss -v +# ./check_hpraid -v # ------------------- # RAID OK: Smart Array 6i in Slot 0 array A logicaldrive 1 (67.8 GB, RAID 1+0, OK) # [Controller Status: OK Cache Status: OK Battery Status: OK] @@ -55,7 +55,7 @@ set -euo pipefail # array A logicaldrive 1 (33.9 GB, RAID 1, Recovering, 26% complete) \ # physicaldrive 1:0 (port 1:id 0 , Parallel SCSI, 36.4 GB, Rebuilding) # -# ./check_cciss -v -p +# ./check_hpraid -v -p # -------------------- # RAID OK: Smart Array 6i in Slot 0 (Embedded) array A logicaldrive 1 (33.9 GB, RAID 1, OK) # physicaldrive 2:0 (port 2:id 0 , Parallel SCSI, 36.4 GB, OK) @@ -73,7 +73,7 @@ set -euo pipefail # physicaldrive 1:0 (port 1:id 0 , Parallel SCSI, 36.4 GB, Rebuilding) \ # physicaldrive 1:1 (port 1:id 1 , Parallel SCSI, 36.4 GB, OK) # -# ./check_cciss -v -b +# ./check_hpraid -v -b # ---------------- # # RAID OK: Smart Array 6i in Slot 0 (Embedded) array A logicaldrive 1 (33.9 GB, RAID 1, OK) [Controller Status: OK] From 0eecc1a4ca57bcf0f273013f4e73c840b1350360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Tue, 16 Jun 2020 10:57:18 +0900 Subject: [PATCH 56/88] nagios-nrpe: check_hpraid: Fixed wrong grep in EXCLUDE_BATTERY --- nagios-nrpe/files/plugins/check_hpraid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nagios-nrpe/files/plugins/check_hpraid b/nagios-nrpe/files/plugins/check_hpraid index 5f140059..4f3a3ed3 100644 --- a/nagios-nrpe/files/plugins/check_hpraid +++ b/nagios-nrpe/files/plugins/check_hpraid @@ -244,8 +244,8 @@ fi # Omit battery/capacitor/cache status check if requested EXCLUDE_BATTERY=${EXCLUDE_BATTERY:-0} if [ "$EXCLUDE_BATTERY" = "1" ]; then - check=$(grep -v 'Battery/Capacitor Status: Failed (Replace Batteries/Capacitors)' "$check") - check=$(grep -v 'Cache Status: Temporarily Disabled' "$check") + check=$(grep -v 'Battery/Capacitor Status: Failed (Replace Batteries/Capacitors)' <<< "$check") + check=$(grep -v 'Cache Status: Temporarily Disabled' <<< "$check") fi check=${check:-} From 4f3e809b7757fe6327609bc1459cac10f1063e79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Tue, 16 Jun 2020 11:16:44 +0900 Subject: [PATCH 57/88] nagios-nrpe: check_hpraid: Use printf for return lines --- nagios-nrpe/files/plugins/check_hpraid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nagios-nrpe/files/plugins/check_hpraid b/nagios-nrpe/files/plugins/check_hpraid index 4f3a3ed3..7181bcd5 100644 --- a/nagios-nrpe/files/plugins/check_hpraid +++ b/nagios-nrpe/files/plugins/check_hpraid @@ -230,7 +230,7 @@ for slot in $slots; do exit "$STATE_UNKNOWN" fi fi - check2="$check2$check2b" + printf -v check2 "%s\n%s" "$check2" "$check2b" if [ "$DEBUG" = "1" ]; then echo "### Get \"physicaldrive\" for slot >>>\n""${check2b}""\n" fi @@ -278,7 +278,7 @@ elif grep -qiE FIRMWARE <<< "$check"; then else if [ "$DEBUG" = "1" ] || [ "$VERBOSE" = "1" ]; then check3=$(grep -E Status <<< "$check") - echo "RAID OK: ${check2} [${check3}]" + printf "RAID OK: %s\n%s\n" "$check2" "$check3" else echo "RAID OK" fi From 725b7ae77b8aabfc8fffbd28ac9e6dea281c8666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Tue, 16 Jun 2020 12:28:10 +0900 Subject: [PATCH 58/88] nagios-nrpe: Add check_hpraid in template --- nagios-nrpe/templates/evolix.cfg.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/nagios-nrpe/templates/evolix.cfg.j2 b/nagios-nrpe/templates/evolix.cfg.j2 index 433a0cd5..3fda958f 100644 --- a/nagios-nrpe/templates/evolix.cfg.j2 +++ b/nagios-nrpe/templates/evolix.cfg.j2 @@ -69,6 +69,7 @@ command[check_varnish]={{ nagios_plugins_directory }}/check_varnish_health -i 12 command[check_haproxy]=sudo {{ nagios_plugins_directory }}/check_haproxy_stats -s /run/haproxy/admin.sock -w 80 -c 90 --ignore-maint --ignore-nolb command[check_minifirewall]=sudo {{ nagios_plugins_directory }}/check_minifirewall command[check_redis_instances]={{ nagios_plugins_directory }}/check_redis_instances +command[check_hpraid]={{ nagios_plugins_directory }}/check_hpraid # Check HTTP "many". Use this to check many websites (http, https, ports, sockets and SSL certificates). # Beware! All checks must not take more than 10s! From 2dc7d22c137513513946aa16a8566c78a94f8125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Tue, 16 Jun 2020 12:34:48 +0900 Subject: [PATCH 59/88] nagios-nrpe: check_hpraid: Update known working RAID controllers --- nagios-nrpe/files/plugins/check_hpraid | 1 + 1 file changed, 1 insertion(+) diff --git a/nagios-nrpe/files/plugins/check_hpraid b/nagios-nrpe/files/plugins/check_hpraid index 7181bcd5..0e6a7729 100644 --- a/nagios-nrpe/files/plugins/check_hpraid +++ b/nagios-nrpe/files/plugins/check_hpraid @@ -23,6 +23,7 @@ set -euo pipefail # # - Adaptec Smart Storage PQI 12G SAS/PCIe 3 (rev 01) # | Smart Array P408i-a SR Gen10 +# | Smart Array P408i-p SR Gen10 # | Smart Array E208i-a SR Gen10 # # From 215ecd6c2c534e95993f952cadf2462ad13bc078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Tue, 16 Jun 2020 12:35:56 +0900 Subject: [PATCH 60/88] evolinux-base: harware: Support HP gen >=10 RAID controller --- evolinux-base/defaults/main.yml | 3 ++ .../{check_hpraid.sh => check_hpraid.cron.sh} | 0 evolinux-base/tasks/hardware.yml | 41 ++++++++++++++----- 3 files changed, 33 insertions(+), 11 deletions(-) rename evolinux-base/files/{check_hpraid.sh => check_hpraid.cron.sh} (100%) diff --git a/evolinux-base/defaults/main.yml b/evolinux-base/defaults/main.yml index 9681f0b7..2a85ecc5 100644 --- a/evolinux-base/defaults/main.yml +++ b/evolinux-base/defaults/main.yml @@ -214,3 +214,6 @@ evolinux_listupgrade_include: True # Generate ldif evolinux_generateldif_include: True + +# Cron check_hpraid +evolinux_cron_checkhpraid_frequency: daily diff --git a/evolinux-base/files/check_hpraid.sh b/evolinux-base/files/check_hpraid.cron.sh similarity index 100% rename from evolinux-base/files/check_hpraid.sh rename to evolinux-base/files/check_hpraid.cron.sh diff --git a/evolinux-base/tasks/hardware.yml b/evolinux-base/tasks/hardware.yml index 628cd6d1..bf7f46b6 100644 --- a/evolinux-base/tasks/hardware.yml +++ b/evolinux-base/tasks/hardware.yml @@ -25,15 +25,17 @@ when: broadcom_netextreme_search.rc == 0 ## RAID - +# Dell and others: MegaRAID SAS +# HP gen <10: Hewlett-Packard Company Smart Array +# HP gen >=10: Adaptec Smart Storage PQI - name: Detect if RAID is installed - shell: lspci | grep "RAID bus controller" | grep -v Intel + shell: lspci -q | grep -e "RAID bus controller" -e "Serial Attached SCSI controller" check_mode: no register: raidmodel changed_when: "'FAILED' in raidmodel.stdout" failed_when: "'FAILED' in raidmodel.stdout" -- name: HP Smart Array package is present +- name: HPE Smart Storage Administrator (ssacli) is present block: - name: Add HPE GPG key apt_key: @@ -44,28 +46,45 @@ apt_repository: repo: 'deb https://downloads.linux.hpe.com/SDR/repo/mcp {{ ansible_distribution_release }}/current non-free' state: present - - - name: Install packages for HP hardware + - name: Install HPE Smart Storage Administrator (ssacli) apt: - name: - - cciss-vol-status - - ssacli + name: ssacli + when: + - "'Hewlett-Packard Company Smart Array' in raidmodel.stdout" + - "'Adaptec Smart Storage PQI' in raidmodel.stdout" + +# NOTE: check_hpraid cron use check_hpraid from nagios-nrpe role +# So, if nagios-nrpe role is not installed it will not work +- name: Install and configure check_hpraid cron (HP gen >=10) + block: + - name: check_hpraid cron is present (HP gen >=10) + copy: + src: check_hpraid.cron.sh + dest: /etc/cron.{{ evolinux_cron_checkhpraid_frequency | mandatory }}/check_hpraid + mode: "0755" + when: "'Adaptec Smart Storage PQI' in raidmodel.stdout" + +- name: Install and configure cciss-vol-status (HP gen <10) + block: + - name: Install cciss-vol-status (HP gen <10) + apt: + name: cciss-vol-status state: present - - name: cciss-vol-statusd init script is present + - name: cciss-vol-statusd init script is present (HP gen <10) template: src: hardware/cciss-vol-statusd.j2 dest: /etc/init.d/cciss-vol-statusd mode: "0755" - - name: Configure cciss-vol-statusd + - name: Configure cciss-vol-statusd (HP gen <10) lineinfile: dest: /etc/default/cciss-vol-statusd line: 'MAILTO="{{ raid_alert_email or general_alert_email | mandatory }}"' regexp: 'MAILTO=' create: yes - - name: Enable HP hardware in systemd + - name: Enable cciss-vol-status in systemd (HP gen <10) service: name: cciss-vol-statusd enabled: true From 0dee37a7e685471ed95ad403491e992cd1c258b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Tue, 16 Jun 2020 12:42:33 +0900 Subject: [PATCH 61/88] evolinux-base/tasks/hardware.yml: Removed trailing whitespace --- evolinux-base/tasks/hardware.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evolinux-base/tasks/hardware.yml b/evolinux-base/tasks/hardware.yml index bf7f46b6..c44c1248 100644 --- a/evolinux-base/tasks/hardware.yml +++ b/evolinux-base/tasks/hardware.yml @@ -76,7 +76,7 @@ src: hardware/cciss-vol-statusd.j2 dest: /etc/init.d/cciss-vol-statusd mode: "0755" - + - name: Configure cciss-vol-statusd (HP gen <10) lineinfile: dest: /etc/default/cciss-vol-statusd From 8a4ae16d73ccb5c9cd4fdd82d5e3eb8e65283032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Tue, 16 Jun 2020 13:20:43 +0900 Subject: [PATCH 62/88] evolinux-base: check_hpraid cron: Add -p --- evolinux-base/files/check_hpraid.cron.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evolinux-base/files/check_hpraid.cron.sh b/evolinux-base/files/check_hpraid.cron.sh index 9c5548c2..d8330f44 100644 --- a/evolinux-base/files/check_hpraid.cron.sh +++ b/evolinux-base/files/check_hpraid.cron.sh @@ -7,7 +7,7 @@ set -euo pipefail TMPDIR=/tmp md5sum=$(command -v md5sum) awk=$(command -v awk) -check_hpraid="/usr/local/lib/nagios/plugins/check_hpraid -v" +check_hpraid="/usr/local/lib/nagios/plugins/check_hpraid -v -p" check_hpraid_output=$(mktemp -p $TMPDIR check_hpraid_XXX) check_hpraid_last="$TMPDIR/check_hpraid_last" trap trapFunc EXIT ERR From fa493699298e0df244de4fa07935f90b5e9b5703 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 16 Jun 2020 13:51:07 +0200 Subject: [PATCH 63/88] varnish: fix start command when multiple addresses are present --- CHANGELOG.md | 1 + varnish/templates/varnish.conf.j2 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ef521e4..5a618aa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ The **patch** part changes incrementally at each release. * lxc-php: Install opensmtpd as intended * mongodb: fix logrotate patterm on Debian buster * evobackup-client: fixed the ssh connection test +* varnish: fix start command when multiple addresses are present ### Removed diff --git a/varnish/templates/varnish.conf.j2 b/varnish/templates/varnish.conf.j2 index 257da952..3020d556 100644 --- a/varnish/templates/varnish.conf.j2 +++ b/varnish/templates/varnish.conf.j2 @@ -2,6 +2,6 @@ [Service] ExecStart= -ExecStart=/usr/sbin/varnishd -F -a {{ varnish_addresses | join(',') }} -T {{ varnish_management_address }} -f {{ varnish_config_file }} -S {{ varnish_secret_file }} -s {{ varnish_storage }} -p thread_pools={{ varnish_thread_pools }} -p thread_pool_add_delay={{ varnish_thread_pool_add_delay }} -p thread_pool_min={{ varnish_thread_pool_min }} -p thread_pool_max={{ varnish_thread_pool_max }} +ExecStart=/usr/sbin/varnishd -F {{ varnish_addresses | map('regex_replace', '^(.*)$', '-a \\1') | list | join(' ') }} -T {{ varnish_management_address }} -f {{ varnish_config_file }} -S {{ varnish_secret_file }} -s {{ varnish_storage }} -p thread_pools={{ varnish_thread_pools }} -p thread_pool_add_delay={{ varnish_thread_pool_add_delay }} -p thread_pool_min={{ varnish_thread_pool_min }} -p thread_pool_max={{ varnish_thread_pool_max }} ExecReload= ExecReload=/etc/varnish/reload-vcl.sh From 952b0d452190ca2b3285b6f1d07eaaecc6a32ab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Wed, 17 Jun 2020 11:25:24 +0900 Subject: [PATCH 64/88] squid: Update regex for sa-update domains. List of domains is like: http://sa-update.dnswl.org/ weight=3 http://www.sa-update.pccc.com/ weight=5 http://sa-update.secnap.net/ weight=5 http://sa-update.space-pro.be/ weight=1 http://sa-update.ena.com/ weight=5 http://sa-update.razx.cloud/ weight=5 http://sa-update.fossies.org/ weight=1 http://sa-update.verein-clean.net/ weight=10 http://sa-update.bitwell.fi/ weight=5 http://sa-update.spamassassin.org/ weight=10 They all start sa-update.*, except for http://www.sa-update.pccc.com/. In that case, we just match sa-update on the domain name. --- squid/files/evolinux-whitelist-defaults.conf | 2 +- squid/files/whitelist-evolinux.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/squid/files/evolinux-whitelist-defaults.conf b/squid/files/evolinux-whitelist-defaults.conf index 70c9e9c1..c0d691c0 100644 --- a/squid/files/evolinux-whitelist-defaults.conf +++ b/squid/files/evolinux-whitelist-defaults.conf @@ -7,7 +7,7 @@ ^hwraid\.le-vert\.net$ ^.*\.clamav\.net$ ^spamassassin\.apache\.org$ -^.*\.sa-update.*$ +^.*sa-update.*$ ^pear\.php\.net$ ^repo\.mysql\.com$ ^deb\.nodesource\.com$ diff --git a/squid/files/whitelist-evolinux.conf b/squid/files/whitelist-evolinux.conf index 07f38a3f..41b81221 100644 --- a/squid/files/whitelist-evolinux.conf +++ b/squid/files/whitelist-evolinux.conf @@ -7,7 +7,7 @@ http://www.kernel.org/.* http://hwraid.le-vert.net/.* http://.*.clamav.net/.* http://spamassassin.apache.org/.* -http://.*.sa-update.* +http://.*sa-update.* http://pear.php.net/.* http://repo.mysql.com/.* From 10385ecf4dcb1fc00cdbbb76aed311358eaf51f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Wed, 17 Jun 2020 12:20:33 +0900 Subject: [PATCH 65/88] nagios-nrpe: check_amavis: Update regex I just installed a Debian Stretch with a pack mail and the check_amavis was not checking the right regex. Amavis is returning: 2.7.0 Ok, discarded, id=17556-09 - INFECTED: Eicar-Signature So the regex should be: -if ($result =~/2.7.0 Ok, discarded, id=[^,]+ - INFECTED: Eicar-Test-Signature/) { +if ($result =~/2.7.0 Ok, discarded, id=\S+ - INFECTED: Eicar-Signature/) { --- nagios-nrpe/files/plugins/check_amavis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nagios-nrpe/files/plugins/check_amavis b/nagios-nrpe/files/plugins/check_amavis index 306d7f8e..8cfc01b4 100755 --- a/nagios-nrpe/files/plugins/check_amavis +++ b/nagios-nrpe/files/plugins/check_amavis @@ -65,7 +65,7 @@ $smtp->close(); print "$result\n"; -if ($result =~/2.7.0 Ok, discarded, id=[^,]+ - INFECTED: Eicar-Test-Signature/) { +if ($result =~/2.7.0 Ok, discarded, id=\S+ - INFECTED: Eicar-Signature/) { print "OK - All fine\n"; exit 0; } else { From 14d8eaac2f1e19b8ea73cf25d88434b7c2686142 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 22 Jun 2020 19:02:29 +0200 Subject: [PATCH 66/88] haproxy: rotate logs with date extension and immediate compression --- CHANGELOG.md | 1 + haproxy/tasks/main.yml | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a618aa3..0b0d7f32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ The **patch** part changes incrementally at each release. * haproxy: chroot and socket path are configurable * haproxy: adapt backports installed package list to distibution * haproxy: split stats variables +* haproxy: rotate logs with date extension and immediate compression * nginx: read server-status values before changing the config * redis: create sudoers file if missing * redis: new syntax for match filter diff --git a/haproxy/tasks/main.yml b/haproxy/tasks/main.yml index 04f08d39..78929bb0 100644 --- a/haproxy/tasks/main.yml +++ b/haproxy/tasks/main.yml @@ -101,4 +101,24 @@ - haproxy - config +- name: Rotate logs with dateext + lineinfile: + dest: /etc/logrotate.d/haproxy + line: ' dateext' + regexp: '^\s*#*\s*(no)?dateext' + insertbefore: '}' + tags: + - haproxy + - config + +- name: Rotate logs with nodelaycompress + lineinfile: + dest: /etc/logrotate.d/haproxy + line: ' nodelaycompress' + regexp: '^\s*#*\s*(no)?delaycompress' + insertbefore: '}' + tags: + - haproxy + - config + - include: munin.yml From fe1756d22a50a44e951f020c2c130ee9ee28b93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Wed, 24 Jun 2020 18:57:08 +0900 Subject: [PATCH 67/88] evolinux-base: check_hpraid.cron.sh: Better logic and use mail First step is to detect errors Second step is to detect different state Added mail comand to replace cron output --- evolinux-base/files/check_hpraid.cron.sh | 66 +++++++++++++++++------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/evolinux-base/files/check_hpraid.cron.sh b/evolinux-base/files/check_hpraid.cron.sh index d8330f44..442b53ba 100644 --- a/evolinux-base/files/check_hpraid.cron.sh +++ b/evolinux-base/files/check_hpraid.cron.sh @@ -10,6 +10,18 @@ awk=$(command -v awk) check_hpraid="/usr/local/lib/nagios/plugins/check_hpraid -v -p" check_hpraid_output=$(mktemp -p $TMPDIR check_hpraid_XXX) check_hpraid_last="$TMPDIR/check_hpraid_last" +# set to false to use cron output (MAILTO) +# otherwise send output with mail command +use_mail=true +body=$(mktemp --tmpdir=/tmp check_hpraid_XXX) +clientmail=$(grep EVOMAINTMAIL /etc/evomaintenance.cf | cut -d'=' -f2) +hostname=$(grep HOSTNAME /etc/evomaintenance.cf | cut -d'=' -f2) +hostname=${hostname%%.evolix.net} +# If hostname is composed with -, remove the first part. +if [[ $hostname =~ "-" ]]; then + hostname=$(echo "$hostname" | cut -d'-' -f2-) +fi + trap trapFunc EXIT ERR testDeps() { @@ -25,6 +37,23 @@ main() { else error=false fi + + # If check_hpraid returned error, display output, save status and + # exit + if $error; then + cp "$check_hpraid_output" "$check_hpraid_last" + if $use_mail; then + mail -s "RAID error on $hostname" "$clientmail" \ + <<< "$check_hpraid_output" + else + cat "$check_hpraid_output" + fi + exit 1 + else + cp "$check_hpraid_output" $check_hpraid_last + exit 0 + fi + if [ ! -f $check_hpraid_last ]; then cp "$check_hpraid_output" $check_hpraid_last fi @@ -34,32 +63,31 @@ main() { md5_now=$(md5sum "$check_hpraid_output" | awk '{print $1}') md5_last=$(md5sum $check_hpraid_last | awk '{print $1}') if [[ "$md5_now" != "$md5_last" ]]; then - cat << EOT - Different RAID state detected. - Was: - $(cat $check_hpraid_last) - Is now: - $(cat $check_hpraid_output) + cat << EOT > "$body" +Different RAID state detected. + +Was: +$(sed 's/^/> /g' "$check_hpraid_last") + +########################### + +Is now: +$(sed 's/^/> /g' "$check_hpraid_output") EOT - cp "$check_hpraid_output" $check_hpraid_last + if $use_mail; then + mail -s "RAID status is different on $hostname" \ + "$clientmail" <<< "$body" + else + cat "$body" + fi + cp "$check_hpraid_output" "$check_hpraid_last" exit 1 fi - - # If check_hpraid returned error, display output, save status and - # exit - if $error; then - cp "$check_hpraid_output" $check_hpraid_last - cat "$check_hpraid_output" - exit 1 - else - cp "$check_hpraid_output" $check_hpraid_last - exit 0 - fi } trapFunc() { - rm "$check_hpraid_output" + rm "$check_hpraid_output" "$body" } testDeps From fd57b17980bd1aef848c863512226aeb904991ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Fri, 26 Jun 2020 17:57:50 +0900 Subject: [PATCH 68/88] evolinux-base: check_hpraid.cron.sh: Fixed wrong else The logic was wrong, an else part was not necessary. --- evolinux-base/files/check_hpraid.cron.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/evolinux-base/files/check_hpraid.cron.sh b/evolinux-base/files/check_hpraid.cron.sh index 442b53ba..074e88a4 100644 --- a/evolinux-base/files/check_hpraid.cron.sh +++ b/evolinux-base/files/check_hpraid.cron.sh @@ -49,9 +49,6 @@ main() { cat "$check_hpraid_output" fi exit 1 - else - cp "$check_hpraid_output" $check_hpraid_last - exit 0 fi if [ ! -f $check_hpraid_last ]; then From 510154724281f74578f4c8fe5a52bf55061a9fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Tue, 30 Jun 2020 04:10:03 +0200 Subject: [PATCH 69/88] generate-ldif: Add NVMe disk support --- generate-ldif/templates/generateldif.sh.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/generate-ldif/templates/generateldif.sh.j2 b/generate-ldif/templates/generateldif.sh.j2 index da37817f..5488d435 100755 --- a/generate-ldif/templates/generateldif.sh.j2 +++ b/generate-ldif/templates/generateldif.sh.j2 @@ -71,7 +71,7 @@ if (test -b /dev/vda); then sdaModel="Virtual VirtIO Disk" elif [ -d /proc/vz ] && [ ! -d /proc/bc ]; then sdaModel="OpenVZ SIMFS disk" -else +elif (lsblk -d -r -n -o TYPE,SIZE,PATH | grep -q sda); then hdparm -I /dev/sda 2>&1 | grep -q bad if [ $? -eq 0 ]; then if (test -n "${raidModel}"); then @@ -82,6 +82,9 @@ else else sdaModel=$(hdparm -I /dev/sda | grep Model | tr -s '\t' ' ' | cut -d' ' -f4-) fi +# hdparm does not support NVME, use smartctl +elif (lsblk -d -r -n -o TYPE,SIZE,PATH | grep -q nvme); then + sdaModel="SSD NVMe: $(smartctl -a /dev/nvme0n1 | grep "Model Number" | tr -s ' ' | cut -d' ' -f3-)" fi ldif_file="/root/${EvoComputerName}.$(date +"%Y%m%d%H%M%S").ldif" From 8b34bec22bb0da3f26673c8a086c7325a50f9676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Tue, 30 Jun 2020 04:36:04 +0200 Subject: [PATCH 70/88] generate-ldif: Skip some odd ethernet devices --- generate-ldif/templates/generateldif.sh.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/generate-ldif/templates/generateldif.sh.j2 b/generate-ldif/templates/generateldif.sh.j2 index 5488d435..7760fbcc 100755 --- a/generate-ldif/templates/generateldif.sh.j2 +++ b/generate-ldif/templates/generateldif.sh.j2 @@ -276,7 +276,10 @@ for net in $(ls /sys/class/net); do echo $path | grep -q virtual if [ $? -ne 0 ]; then hw=$(cat ${path}/address) + # In some cases some devices does not have a vendor or device, skip it + test -f ${path}/device/vendor || continue vendor_id=$(cat ${path}/device/vendor) + test -f ${path}/device/device || continue dev_id=$(cat ${path}/device/device) [ "${dev_id}" = "0x0001" ] && dev_id="0x1000" dev=$(lspci -d "${vendor_id}:${dev_id}" -vm) From 63bfef13eee2c488c058886986b82b03901f8209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Tue, 30 Jun 2020 05:11:05 +0200 Subject: [PATCH 71/88] generate-ldif: Patched computerOS detection --- generate-ldif/templates/generateldif.sh.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/generate-ldif/templates/generateldif.sh.j2 b/generate-ldif/templates/generateldif.sh.j2 index 7760fbcc..d5c19411 100755 --- a/generate-ldif/templates/generateldif.sh.j2 +++ b/generate-ldif/templates/generateldif.sh.j2 @@ -25,7 +25,8 @@ EvoComputerName=$(hostname -s) dnsPTRrecord=$(hostname -f) HardwareMark=$(dmidecode -s system-manufacturer | grep -v '^#') computerIP=$(hostname -i | cut -d' ' -f1) -computerOS=$(lsb_release -s -d | sed 's#\..##') +# The sed part does not works for squeeze and previous +computerOS=$(lsb_release -s -d | sed -E 's#\.[0-9]{1,}##') computerKernel=$(uname -r) HardwareSerial=$(dmidecode -s system-serial-number | grep -v '^#') From 0c9b9b2628d3aae71d0e1182cd12cfa5fa250970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Wed, 1 Jul 2020 10:18:30 +0900 Subject: [PATCH 72/88] evolinux-base: check_hpraid.cron.sh: Fixed wrong `<<<` usage --- evolinux-base/files/check_hpraid.cron.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evolinux-base/files/check_hpraid.cron.sh b/evolinux-base/files/check_hpraid.cron.sh index 074e88a4..bd181716 100644 --- a/evolinux-base/files/check_hpraid.cron.sh +++ b/evolinux-base/files/check_hpraid.cron.sh @@ -44,7 +44,7 @@ main() { cp "$check_hpraid_output" "$check_hpraid_last" if $use_mail; then mail -s "RAID error on $hostname" "$clientmail" \ - <<< "$check_hpraid_output" + < "$check_hpraid_output" else cat "$check_hpraid_output" fi @@ -73,7 +73,7 @@ $(sed 's/^/> /g' "$check_hpraid_output") EOT if $use_mail; then mail -s "RAID status is different on $hostname" \ - "$clientmail" <<< "$body" + "$clientmail" < "$body" else cat "$body" fi From ec95f9538f4fd9008194c75812f67443d8b62d10 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 17 Jul 2020 13:48:18 +0200 Subject: [PATCH 73/88] mongodb: install custom munin plugins --- CHANGELOG.md | 11 +-- mongodb/files/munin/mongo_btree | 63 +++++++++++++++ mongodb/files/munin/mongo_collections | 106 ++++++++++++++++++++++++++ mongodb/files/munin/mongo_conn | 57 ++++++++++++++ mongodb/files/munin/mongo_docs | 72 +++++++++++++++++ mongodb/files/munin/mongo_lock | 56 ++++++++++++++ mongodb/files/munin/mongo_mem | 62 +++++++++++++++ mongodb/files/munin/mongo_ops | 58 ++++++++++++++ mongodb/files/munin/mongo_page_faults | 57 ++++++++++++++ mongodb/handlers/main.yml | 5 ++ mongodb/tasks/main_buster.yml | 33 ++++++++ 11 files changed, 575 insertions(+), 5 deletions(-) create mode 100755 mongodb/files/munin/mongo_btree create mode 100755 mongodb/files/munin/mongo_collections create mode 100755 mongodb/files/munin/mongo_conn create mode 100755 mongodb/files/munin/mongo_docs create mode 100755 mongodb/files/munin/mongo_lock create mode 100755 mongodb/files/munin/mongo_mem create mode 100755 mongodb/files/munin/mongo_ops create mode 100755 mongodb/files/munin/mongo_page_faults diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b0d7f32..22acb022 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,20 +29,21 @@ The **patch** part changes incrementally at each release. ### Changed -* lxc-php: Do --no-install-recommends for ssmtp/opensmtpd -* packweb-apache: Don't turn on mod-evasive emails by default +* certbot: install certbot dependencies non-interactively for jessie +* evoacme: upstream release 20.06.1 +* evoacme: read values from environment before defaults file * haproxy: deport SSL tuning to Mozilla SSL generator * haproxy: chroot and socket path are configurable * haproxy: adapt backports installed package list to distibution * haproxy: split stats variables * haproxy: rotate logs with date extension and immediate compression +* lxc-php: Do --no-install-recommends for ssmtp/opensmtpd +* mongodb: install custom munin plugins * nginx: read server-status values before changing the config +* packweb-apache: Don't turn on mod-evasive emails by default * redis: create sudoers file if missing * redis: new syntax for match filter * redis: raise an error is port 6379 is used in "instance" mode -* evoacme: upstream release 20.06.1 -* evoacme: read values from environment before defaults file -* certbot: install certbot dependencies non-interactively for jessie ### Fixed diff --git a/mongodb/files/munin/mongo_btree b/mongodb/files/munin/mongo_btree new file mode 100755 index 00000000..32173814 --- /dev/null +++ b/mongodb/files/munin/mongo_btree @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os +import pymongo + + +def getClient(): + if 'MONGO_DB_URI' in os.environ: + return pymongo.MongoClient(os.environ['MONGO_DB_URI']) + else: + return pymongo.MongoClient() + + +def getServerStatus(): + c = getClient() + return c.admin.command('serverStatus', workingSet=True) + +def get(): + return getServerStatus()["indexCounters"] + +def doData(): + for k,v in get().iteritems(): + print( str(k) + ".value " + str(int(v)) ) + +def doConfig(): + + print "graph_title MongoDB btree stats" + print "graph_args --base 1000 -l 0" + print "graph_vlabel mb ${graph_period}" + print "graph_category MongoDB" + + for k in get(): + print k + ".label " + k + print k + ".min 0" + print k + ".type COUNTER" + print k + ".max 500000" + print k + ".draw LINE1" + + + + + + +if __name__ == "__main__": + + from os import environ + if 'HOST' in environ: + host = environ['HOST'] + if 'PORT' in environ: + port = environ['PORT'] + if 'USER' in environ: + user = environ['USER'] + if 'PASSWORD' in environ: + password = environ['PASSWORD'] + +if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() +else: + doData() diff --git a/mongodb/files/munin/mongo_collections b/mongodb/files/munin/mongo_collections new file mode 100755 index 00000000..ca70c715 --- /dev/null +++ b/mongodb/files/munin/mongo_collections @@ -0,0 +1,106 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os +import pymongo + + +def getClient(): + if 'MONGO_DB_URI' in os.environ: + return pymongo.MongoClient(os.environ['MONGO_DB_URI']) + else: + return pymongo.MongoClient() + + +def getServerStatus(): + c = getClient() + return c.admin.command('serverStatus', workingSet=True) +import re +FIELD_ESCAPE = re.compile("[^A-Za-z_]") + + +def escape_field(name): + return FIELD_ESCAPE.sub("_", name) + + +def need_multigraph(): + if 'MUNIN_CAP_MULTIGRAPH' not in os.environ: + sys.stderr.write('MUNIN_CAP_MULTIGRAPH not found in environment\n') + sys.exit(1) + + +def collections(include_stats=False): + c = getClient() + for db in c.database_names(): + for collection in c[db].collection_names(): + name = db + "." + collection + if include_stats: + yield name, c[db].command("collstats", collection) + else: + yield name + + +def doData(): + need_multigraph() + data = list(collections(True)) + + print "multigraph collection_count" + for name, stats in data: + print(escape_field(name) + ".value " + str(stats["count"])) + + print "multigraph collection_size" + for name, stats in data: + print(escape_field(name) + ".value " + str(stats["size"])) + + +def doConfig(): + need_multigraph() + names = list(collections()) + + print "multigraph collection_count" + print "graph_title MongoDB collection document count" + print "graph_args --base 1000 -l 0" + print "graph_vlabel collection document count" + print "graph_category MongoDB" + print "graph_total total" + + for name in names: + field_name = escape_field(name) + print field_name + ".label " + name + print field_name + ".min 0" + print field_name + ".type GAUGE" + print field_name + ".draw LINE1" + + print "multigraph collection_size" + print "graph_title MongoDB collection size" + print "graph_args --base 1024 -l 0" + print "graph_vlabel collection size" + print "graph_category MongoDB" + print "graph_total total" + + for name in names: + field_name = escape_field(name) + print field_name + ".label " + name + print field_name + ".min 0" + print field_name + ".type GAUGE" + print field_name + ".draw LINE1" + +if __name__ == "__main__": + + from os import environ + if 'HOST' in environ: + host = environ['HOST'] + if 'PORT' in environ: + port = environ['PORT'] + if 'USER' in environ: + user = environ['USER'] + if 'PASSWORD' in environ: + password = environ['PASSWORD'] + +if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() +else: + doData() diff --git a/mongodb/files/munin/mongo_conn b/mongodb/files/munin/mongo_conn new file mode 100755 index 00000000..fc8e5f3c --- /dev/null +++ b/mongodb/files/munin/mongo_conn @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os +import pymongo + + +def getClient(): + if 'MONGO_DB_URI' in os.environ: + return pymongo.MongoClient(os.environ['MONGO_DB_URI']) + else: + return pymongo.MongoClient() + + +def getServerStatus(): + c = getClient() + return c.admin.command('serverStatus', workingSet=True) + +name = "connections" + + +def doData(): + print name + ".value " + str( getServerStatus()["connections"]["current"] ) + +def doConfig(): + + print "graph_title MongoDB current connections" + print "graph_args --base 1000 -l 0" + print "graph_vlabel connections" + print "graph_category MongoDB" + + print name + ".label " + name + + + + + + +if __name__ == "__main__": + + from os import environ + if 'HOST' in environ: + host = environ['HOST'] + if 'PORT' in environ: + port = environ['PORT'] + if 'USER' in environ: + user = environ['USER'] + if 'PASSWORD' in environ: + password = environ['PASSWORD'] + +if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() +else: + doData() diff --git a/mongodb/files/munin/mongo_docs b/mongodb/files/munin/mongo_docs new file mode 100755 index 00000000..f8daa21f --- /dev/null +++ b/mongodb/files/munin/mongo_docs @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os +import pymongo + + +def getClient(): + if 'MONGO_DB_URI' in os.environ: + return pymongo.MongoClient(os.environ['MONGO_DB_URI']) + else: + return pymongo.MongoClient() + + +def getServerStatus(): + c = getClient() + return c.admin.command('serverStatus', workingSet=True) + +def getDatabasesStats(): + host = "127.0.0.1" + port = 27017 + c = getClient() + + dbs = {} + for k in c.database_names(): + if k != "admin" and k != "local" and k != "": + db = c[k] + dbs[k] = {} + for coll in db.collection_names(): + if '.' not in coll: + dbs[k][coll] = db[coll].count() + + return dbs + +def doData(): + ss = getDatabasesStats() + for k,v in ss.iteritems(): + for a,b in v.iteritems(): + print(str(k)+str(a) + ".value " + str(b)) + + +def doConfig(): + + print "graph_title MongoDB documents count" + print "graph_args --base 1000 -l 0 --vertical-label Docs" + print "graph_category MongoDB" + + ss = getDatabasesStats() + for k,v in ss.iteritems(): + for a,b in v.iteritems(): + print str(k)+str(a) + ".label " + str(k) + " " + str(a) + print str(k)+str(a) + ".draw LINE1" + +if __name__ == "__main__": + + from os import environ + if 'HOST' in environ: + host = environ['HOST'] + if 'PORT' in environ: + port = environ['PORT'] + if 'USER' in environ: + user = environ['USER'] + if 'PASSWORD' in environ: + password = environ['PASSWORD'] + +if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() +else: + doData() diff --git a/mongodb/files/munin/mongo_lock b/mongodb/files/munin/mongo_lock new file mode 100755 index 00000000..8d219d73 --- /dev/null +++ b/mongodb/files/munin/mongo_lock @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os +import pymongo + + +def getClient(): + if 'MONGO_DB_URI' in os.environ: + return pymongo.MongoClient(os.environ['MONGO_DB_URI']) + else: + return pymongo.MongoClient() + + +def getServerStatus(): + c = getClient() + return c.admin.command('serverStatus', workingSet=True) + +name = "locked" + +def doData(): + print name + ".value " + str( 100 * (getServerStatus()["globalLock"]["lockTime"]/getServerStatus()["globalLock"]["totalTime"]) ) + +def doConfig(): + + print "graph_title MongoDB global write lock percentage" + print "graph_args --base 1000 -l 0 " + print "graph_vlabel percentage" + print "graph_category MongoDB" + + print name + ".label " + name + + + + + + +if __name__ == "__main__": + + from os import environ + if 'HOST' in environ: + host = environ['HOST'] + if 'PORT' in environ: + port = environ['PORT'] + if 'USER' in environ: + user = environ['USER'] + if 'PASSWORD' in environ: + password = environ['PASSWORD'] + +if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() +else: + doData() diff --git a/mongodb/files/munin/mongo_mem b/mongodb/files/munin/mongo_mem new file mode 100755 index 00000000..0a5ca22c --- /dev/null +++ b/mongodb/files/munin/mongo_mem @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os +import pymongo + + +def getClient(): + if 'MONGO_DB_URI' in os.environ: + return pymongo.MongoClient(os.environ['MONGO_DB_URI']) + else: + return pymongo.MongoClient() + + +def getServerStatus(): + c = getClient() + return c.admin.command('serverStatus', workingSet=True) + +def ok(s): + return s == "resident" or s == "virtual" or s == "mapped" + +def doData(): + for k,v in getServerStatus()["mem"].iteritems(): + if ok(k): + print( str(k) + ".value " + str(v * 1024 * 1024) ) + +def doConfig(): + + print "graph_title MongoDB memory usage" + print "graph_args --base 1024 -l 0 --vertical-label Bytes" + print "graph_category MongoDB" + + for k in getServerStatus()["mem"]: + if ok( k ): + print k + ".label " + k + print k + ".draw LINE1" + + + + + + + +if __name__ == "__main__": + + from os import environ + if 'HOST' in environ: + host = environ['HOST'] + if 'PORT' in environ: + port = environ['PORT'] + if 'USER' in environ: + user = environ['USER'] + if 'PASSWORD' in environ: + password = environ['PASSWORD'] + +if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() +else: + doData() diff --git a/mongodb/files/munin/mongo_ops b/mongodb/files/munin/mongo_ops new file mode 100755 index 00000000..4a29b411 --- /dev/null +++ b/mongodb/files/munin/mongo_ops @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os +import pymongo + + +def getClient(): + if 'MONGO_DB_URI' in os.environ: + return pymongo.MongoClient(os.environ['MONGO_DB_URI']) + else: + return pymongo.MongoClient() + + +def getServerStatus(): + c = getClient() + return c.admin.command('serverStatus', workingSet=True) + + +def doData(): + ss = getServerStatus() + for k,v in ss["opcounters"].iteritems(): + print( str(k) + ".value " + str(v) ) + +def doConfig(): + + print "graph_title MongoDB ops" + print "graph_args --base 1000 -l 0" + print "graph_vlabel ops / ${graph_period}" + print "graph_category MongoDB" + print "graph_total total" + + for k in getServerStatus()["opcounters"]: + print k + ".label " + k + print k + ".min 0" + print k + ".type COUNTER" + print k + ".max 500000" + print k + ".draw LINE1" + +if __name__ == "__main__": + + from os import environ + if 'HOST' in environ: + host = environ['HOST'] + if 'PORT' in environ: + port = environ['PORT'] + if 'USER' in environ: + user = environ['USER'] + if 'PASSWORD' in environ: + password = environ['PASSWORD'] + +if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() +else: + doData() diff --git a/mongodb/files/munin/mongo_page_faults b/mongodb/files/munin/mongo_page_faults new file mode 100755 index 00000000..8d96cf2b --- /dev/null +++ b/mongodb/files/munin/mongo_page_faults @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +## GENERATED FILE - DO NOT EDIT + +import urllib2 +import sys +import os +import pymongo + +def getServerStatus(): + if 'MONGO_DB_URI' in os.environ: + c = pymongo.MongoClient(os.environ['MONGO_DB_URI']) + else: + c = pymongo.MongoClient() + + return c.admin.command('serverStatus', workingSet=True) + +name = "page_faults" + +def get(): + return getServerStatus()["extra_info"][name] + + +def doData(): + print(name + ".value " + str(get())) + + +def doConfig(): + + print "graph_title MongoDB page faults" + print "graph_args --base 1000 -l 0" + print "graph_vlabel faults / ${graph_period}" + print "graph_category MongoDB" + print "graph_total total" + + print name + ".label " + name + print name + ".min 0" + print name + ".type COUNTER" + print name + ".max 10000" + print name + ".draw LINE1" + +if __name__ == "__main__": + + from os import environ + if 'HOST' in environ: + host = environ['HOST'] + if 'PORT' in environ: + port = environ['PORT'] + if 'USER' in environ: + user = environ['USER'] + if 'PASSWORD' in environ: + password = environ['PASSWORD'] + +if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() +else: + doData() diff --git a/mongodb/handlers/main.yml b/mongodb/handlers/main.yml index 62ff5f28..15f70437 100644 --- a/mongodb/handlers/main.yml +++ b/mongodb/handlers/main.yml @@ -9,3 +9,8 @@ service: name: mongodb state: restarted + +- name: restart munin-node + systemd: + name: munin-node + state: restarted diff --git a/mongodb/tasks/main_buster.yml b/mongodb/tasks/main_buster.yml index b04047e0..253fde91 100644 --- a/mongodb/tasks/main_buster.yml +++ b/mongodb/tasks/main_buster.yml @@ -44,3 +44,36 @@ dest: /etc/logrotate.d/mongodb force: yes backup: no + +- name: Munin plugins are present + copy: + src: "munin/{{ item }}" + dest: '/usr/local/share/munin/plugins/{{ item }}' + force: yes + state: present + with_items: + - mongo_btree + - mongo_collections + - mongo_conn + - mongo_docs + - mongo_lock + - mongo_mem + - mongo_ops + - mongo_page_faults + notify: restart munin-node + +- name: Enable core Munin plugins + file: + src: '/usr/local/share/munin/plugins/{{ item }}' + dest: /etc/munin/plugins/{{ item }} + state: link + with_items: + - mongo_btree + - mongo_collections + - mongo_conn + - mongo_docs + - mongo_lock + - mongo_mem + - mongo_ops + - mongo_page_faults + notify: restart munin-node From 1063d2dc1c84b9510edb6d941e9535b57e54432a Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 17 Jul 2020 13:49:07 +0200 Subject: [PATCH 74/88] elasticsearch is compatible with buster --- elasticsearch/meta/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/elasticsearch/meta/main.yml b/elasticsearch/meta/main.yml index 407fbc76..736c0a42 100644 --- a/elasticsearch/meta/main.yml +++ b/elasticsearch/meta/main.yml @@ -14,6 +14,7 @@ galaxy_info: versions: - jessie - stretch + - buster galaxy_tags: [] # List tags for your role here, one per line. A tag is From f2f63eee3656f97ec2cf5c5f760a6d7230839849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 19 Jul 2020 11:30:00 +0200 Subject: [PATCH 75/88] elasticsearch: set tmpdir before datadir --- CHANGELOG.md | 1 + elasticsearch/tasks/main.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22acb022..da1e5aa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The **patch** part changes incrementally at each release. ### Changed * certbot: install certbot dependencies non-interactively for jessie +* elasticsearch: set tmpdir before datadir * evoacme: upstream release 20.06.1 * evoacme: read values from environment before defaults file * haproxy: deport SSL tuning to Mozilla SSL generator diff --git a/elasticsearch/tasks/main.yml b/elasticsearch/tasks/main.yml index 361b15dc..126170a1 100644 --- a/elasticsearch/tasks/main.yml +++ b/elasticsearch/tasks/main.yml @@ -6,10 +6,10 @@ - include: bootstrap_checks.yml -- include: datadir.yml - - include: tmpdir.yml +- include: datadir.yml + - include: logs.yml - include: additional_scripts.yml From 43122a5ea9a117ce7fbd203d489f3e24752738b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 19 Jul 2020 11:40:43 +0200 Subject: [PATCH 76/88] elasticsearch: configure cluster with seed hosts and initial masters --- CHANGELOG.md | 1 + elasticsearch/defaults/main.yml | 5 ++++- elasticsearch/tasks/configuration.yml | 26 +++++++++++++++++++++----- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da1e5aa6..00d18088 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The **patch** part changes incrementally at each release. * certbot: install certbot dependencies non-interactively for jessie * elasticsearch: set tmpdir before datadir +* elasticsearch: configure cluster with seed hosts and initial masters * evoacme: upstream release 20.06.1 * evoacme: read values from environment before defaults file * haproxy: deport SSL tuning to Mozilla SSL generator diff --git a/elasticsearch/defaults/main.yml b/elasticsearch/defaults/main.yml index 91e81915..eb254807 100644 --- a/elasticsearch/defaults/main.yml +++ b/elasticsearch/defaults/main.yml @@ -5,9 +5,12 @@ elasticsearch_cluster_name: Null elasticsearch_cluster_members: Null elasticsearch_minimum_master_nodes: Null elasticsearch_node_name: "${HOSTNAME}" -elasticsearch_network_host: "[_local_]" +elasticsearch_network_host: + - "_local_" elasticsearch_network_publish_host: Null elasticsearch_http_publish_host: Null +elasticsearch_discovery_seed_hosts: Null +elasticsearch_cluster_initial_master_nodes: Null elasticsearch_custom_datadir: Null elasticsearch_custom_tmpdir: Null elasticsearch_default_tmpdir: /var/lib/elasticsearch/tmp diff --git a/elasticsearch/tasks/configuration.yml b/elasticsearch/tasks/configuration.yml index 50e8c98c..d4130bae 100644 --- a/elasticsearch/tasks/configuration.yml +++ b/elasticsearch/tasks/configuration.yml @@ -22,7 +22,7 @@ - name: Configure network host lineinfile: dest: /etc/elasticsearch/elasticsearch.yml - line: "network.host: {{ elasticsearch_network_host }}" + line: "network.host: {{ elasticsearch_network_host | to_yaml }}" regexp: "^network.host:" insertafter: "^# *network.host:" when: elasticsearch_network_host|default("", True) @@ -32,7 +32,7 @@ - name: Configure network publish_host lineinfile: dest: /etc/elasticsearch/elasticsearch.yml - line: "network.publish_host: {{ elasticsearch_network_publish_host }}" + line: "network.publish_host: {{ elasticsearch_network_publish_host | to_yaml }}" regexp: "^network.publish_host:" insertafter: "^network.host:" when: elasticsearch_network_publish_host|default("", True) @@ -42,13 +42,31 @@ - name: Configure http publish_host lineinfile: dest: /etc/elasticsearch/elasticsearch.yml - line: "http.publish_host: {{ elasticsearch_http_publish_host }}" + line: "http.publish_host: {{ elasticsearch_http_publish_host | to_yaml }}" regexp: "^http.publish_host:" insertafter: "^http.port:" when: elasticsearch_http_publish_host|default("", True) tags: - config +- name: Configure discovery seed hosts + lineinfile: + dest: /etc/elasticsearch/elasticsearch.yml + line: "discovery.seed_hosts: {{ elasticsearch_discovery_seed_hosts | to_yaml }}" + regexp: "^discovery.seed_hosts:" + when: elasticsearch_discovery_seed_hosts + tags: + - config + +- name: Configure initial master nodes + lineinfile: + dest: /etc/elasticsearch/elasticsearch.yml + line: "cluster.initial_master_nodes: {{ elasticsearch_cluster_initial_master_nodes | to_yaml }}" + regexp: "^cluster.initial_master_nodes:" + when: elasticsearch_cluster_initial_master_nodes + tags: + - config + - name: Configure RESTART_ON_UPGRADE lineinfile: dest: /etc/default/elasticsearch @@ -93,5 +111,3 @@ when: elasticsearch_minimum_master_nodes|default("", True) tags: - config - - From c89d6995186f28a16c53790f846cf8d70648f667 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 21 Jul 2020 10:45:34 +0200 Subject: [PATCH 77/88] certbot: fix haproxy hook (ssl cert directory detection) It was matching additional parameters. Now it matches on the first argument after "crt" --- CHANGELOG.md | 1 + certbot/files/hooks/haproxy.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00d18088..5c76d489 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ The **patch** part changes incrementally at each release. ### Changed +* certbot: fix haproxy hook (ssl cert directory detection) * certbot: install certbot dependencies non-interactively for jessie * elasticsearch: set tmpdir before datadir * elasticsearch: configure cluster with seed hosts and initial masters diff --git a/certbot/files/hooks/haproxy.sh b/certbot/files/hooks/haproxy.sh index 4998d55f..1a7f5d4e 100644 --- a/certbot/files/hooks/haproxy.sh +++ b/certbot/files/hooks/haproxy.sh @@ -36,7 +36,7 @@ cert_and_key_mismatch() { } detect_haproxy_cert_dir() { # get last field or line wich defines the crt directory - config_cert_dir=$(grep -r -o -E -h '^\s*bind .* crt /etc/.+\b' "${haproxy_config_file}" | head -1 | awk '{ print $(NF)}') + config_cert_dir=$(grep -r -o -E -h '^\s*bind .* crt /etc/\S+' "${haproxy_config_file}" | head -1 | awk '{ print $(NF)}') if [ -n "${config_cert_dir}" ]; then debug "Cert directory is configured with ${config_cert_dir}" echo "${config_cert_dir}" From f55e5be2e554154eadd709786ba681dfbf78ddd6 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 21 Jul 2020 10:46:34 +0200 Subject: [PATCH 78/88] elasticsearch: keep native values --- elasticsearch/tasks/configuration.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elasticsearch/tasks/configuration.yml b/elasticsearch/tasks/configuration.yml index d4130bae..e8362fa3 100644 --- a/elasticsearch/tasks/configuration.yml +++ b/elasticsearch/tasks/configuration.yml @@ -22,7 +22,7 @@ - name: Configure network host lineinfile: dest: /etc/elasticsearch/elasticsearch.yml - line: "network.host: {{ elasticsearch_network_host | to_yaml }}" + line: "network.host: {{ elasticsearch_network_host }}" regexp: "^network.host:" insertafter: "^# *network.host:" when: elasticsearch_network_host|default("", True) @@ -32,7 +32,7 @@ - name: Configure network publish_host lineinfile: dest: /etc/elasticsearch/elasticsearch.yml - line: "network.publish_host: {{ elasticsearch_network_publish_host | to_yaml }}" + line: "network.publish_host: {{ elasticsearch_network_publish_host }}" regexp: "^network.publish_host:" insertafter: "^network.host:" when: elasticsearch_network_publish_host|default("", True) @@ -42,7 +42,7 @@ - name: Configure http publish_host lineinfile: dest: /etc/elasticsearch/elasticsearch.yml - line: "http.publish_host: {{ elasticsearch_http_publish_host | to_yaml }}" + line: "http.publish_host: {{ elasticsearch_http_publish_host }}" regexp: "^http.publish_host:" insertafter: "^http.port:" when: elasticsearch_http_publish_host|default("", True) From 08cdd0cd18b6af8e560bf6bef8ed2bedaefa029e Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Thu, 30 Jul 2020 11:31:19 +0200 Subject: [PATCH 79/88] mongodb: Fix issue introduced by 8aa7f6cf3366606010407ee0af4687c7a716463f --- mongodb/tasks/main_buster.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/mongodb/tasks/main_buster.yml b/mongodb/tasks/main_buster.yml index 253fde91..5aae2ed3 100644 --- a/mongodb/tasks/main_buster.yml +++ b/mongodb/tasks/main_buster.yml @@ -50,7 +50,6 @@ src: "munin/{{ item }}" dest: '/usr/local/share/munin/plugins/{{ item }}' force: yes - state: present with_items: - mongo_btree - mongo_collections From f68a79c022784f4d75a60d4d7c5c0e8d82c2dc57 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 18 Aug 2020 14:00:46 +0200 Subject: [PATCH 80/88] filebeat: allow using a template --- CHANGELOG.md | 1 + filebeat/defaults/main.yml | 11 + filebeat/tasks/main.yml | 83 ++++++- filebeat/templates/filebeat.default.yml.j2 | 247 +++++++++++++++++++++ 4 files changed, 331 insertions(+), 11 deletions(-) create mode 100644 filebeat/templates/filebeat.default.yml.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c76d489..874df943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The **patch** part changes incrementally at each release. ### Added * certbot: detect HAProxy cert directory +* filebeat: allow using a template * haproxy: add deny_ips file to reject connections * haproxy: add some comments to default config * haproxy: enable stats frontend with access lists diff --git a/filebeat/defaults/main.yml b/filebeat/defaults/main.yml index 008daa27..cd92eb3c 100644 --- a/filebeat/defaults/main.yml +++ b/filebeat/defaults/main.yml @@ -4,3 +4,14 @@ elastic_stack_version: "6.x" filebeat_logstash_plugin: False filebeat_processors_cloud_metadata: False + +filebeat_elasticsearch_hosts: + - "localhost:9200" +filebeat_elasticsearch_protocol: "http" +filebeat_elasticsearch_auth_api_key: "" +filebeat_elasticsearch_auth_username: "" +filebeat_elasticsearch_auth_password: "" + +filebeat_use_config_template: False +filebeat_update_config: True +filebeat_force_config: True diff --git a/filebeat/tasks/main.yml b/filebeat/tasks/main.yml index 0aca06d6..34feb5c7 100644 --- a/filebeat/tasks/main.yml +++ b/filebeat/tasks/main.yml @@ -66,18 +66,79 @@ - logstash_plugin.stat.exists - not logstash_plugin_installed | success -- name: cloud_metadata processor is disabled - replace: - dest: /etc/filebeat/filebeat.yml - regexp: '^(\s+)(- add_cloud_metadata:)' - replace: '\1# \2' - notify: restart filebeat - when: not filebeat_processors_cloud_metadata +# When we don't use a config template (default) +- block: + - name: cloud_metadata processor is disabled + replace: + dest: /etc/filebeat/filebeat.yml + regexp: '^(\s+)(- add_cloud_metadata:)' + replace: '\1# \2' + notify: restart filebeat + when: not filebeat_processors_cloud_metadata -- name: cloud_metadata processor is disabled + - name: cloud_metadata processor is disabled + lineinfile: + dest: /etc/filebeat/filebeat.yml + line: " - add_cloud_metadata: ~" + insert_after: '^processors:' + notify: restart filebeat + when: filebeat_processors_cloud_metadata + + - name: Filebeat knows where to find Elasticsearch + lineinfile: + dest: /etc/filebeat/filebeat.yml + regexp: '^ hosts: .*' + line: " hosts: [\"{{ filebeat_elasticsearch_hosts | join('\", \"') }}\"]" + insertafter: "output.elasticsearch:" + notify: restart filebeat + when: + - filebeat_elasticsearch_hosts + + - name: Filebeat protocol for Elasticsearch + lineinfile: + dest: /etc/filebeat/filebeat.yml + regexp: '^ #?protocol: .*' + line: " protocol: \"{{ filebeat_elasticsearch_protocol }}\"" + insertafter: "output.elasticsearch:" + notify: restart filebeat + when: filebeat_elasticsearch_protocol == "http" or filebeat_elasticsearch_protocol == "https" + + - name: Filebeat auth/username for Elasticsearch are configured + lineinfile: + dest: /etc/filebeat/filebeat.yml + regexp: '{{ item.regexp }}' + line: '{{ item.line }}' + insertafter: "output.elasticsearch:" + with_items: + - { regexp: '^ #?username: .*', line: ' username: "{{ filebeat_elasticsearch_auth_username }}"' } + - { regexp: '^ #?password: .*', line: ' password: "{{ filebeat_elasticsearch_auth_password }}"' } + notify: restart filebeat + when: + - filebeat_elasticsearch_auth_username + - filebeat_elasticsearch_auth_password + when: not filebeat_use_config_template + +- name: Filebeat api_key for Elasticsearch are configured lineinfile: dest: /etc/filebeat/filebeat.yml - line: " - add_cloud_metadata: ~" - insert_after: '^processors:' + regexp: '^ #?api_key: .*' + line: ' api_key: "{{ filebeat_elasticsearch_auth_api_key }}"' + insertafter: "output.elasticsearch:" notify: restart filebeat - when: filebeat_processors_cloud_metadata + when: filebeat_elasticsearch_auth_api_key + +# When we use a config template +- block: + - name: Configuration is up-to-date + template: + src: "{{ item }}" + dest: /etc/filebeat/filebeat.yml + force: "{{ filebeat_force_config }}" + with_first_found: + - "templates/filebeat/filebeat.{{ inventory_hostname }}.yml.j2" + - "templates/filebeat/filebeat.{{ host_group }}.yml.j2" + - "templates/filebeat/filebeat.default.yml.j2" + - "filebeat.default.yml.j2" + notify: restart filebeat + when: filebeat_update_config + when: filebeat_use_config_template diff --git a/filebeat/templates/filebeat.default.yml.j2 b/filebeat/templates/filebeat.default.yml.j2 new file mode 100644 index 00000000..65a15fd1 --- /dev/null +++ b/filebeat/templates/filebeat.default.yml.j2 @@ -0,0 +1,247 @@ +###################### Filebeat Configuration Example ######################### + +# This file is an example configuration file highlighting only the most common +# options. The filebeat.reference.yml file from the same directory contains all the +# supported options with more comments. You can use it as a reference. +# +# You can find the full configuration reference here: +# https://www.elastic.co/guide/en/beats/filebeat/index.html + +# For more available modules and options, please see the filebeat.reference.yml sample +# configuration file. + +# ============================== Filebeat inputs =============================== + +filebeat.inputs: + +# Each - is an input. Most options can be set at the input level, so +# you can use different inputs for various configurations. +# Below are the input specific configurations. + +- type: log + + # Change to true to enable this input configuration. + enabled: false + + # Paths that should be crawled and fetched. Glob based paths. + paths: + - /var/log/*.log + #- c:\programdata\elasticsearch\logs\* + + # Exclude lines. A list of regular expressions to match. It drops the lines that are + # matching any regular expression from the list. + #exclude_lines: ['^DBG'] + + # Include lines. A list of regular expressions to match. It exports the lines that are + # matching any regular expression from the list. + #include_lines: ['^ERR', '^WARN'] + + # Exclude files. A list of regular expressions to match. Filebeat drops the files that + # are matching any regular expression from the list. By default, no files are dropped. + #exclude_files: ['.gz$'] + + # Optional additional fields. These fields can be freely picked + # to add additional information to the crawled log files for filtering + #fields: + # level: debug + # review: 1 + + ### Multiline options + + # Multiline can be used for log messages spanning multiple lines. This is common + # for Java Stack Traces or C-Line Continuation + + # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [ + #multiline.pattern: ^\[ + + # Defines if the pattern set under pattern should be negated or not. Default is false. + #multiline.negate: false + + # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern + # that was (not) matched before or after or as long as a pattern is not matched based on negate. + # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash + #multiline.match: after + +# ============================== Filebeat modules ============================== + +filebeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s + +# ======================= Elasticsearch template setting ======================= + +setup.template.settings: + index.number_of_shards: 1 + #index.codec: best_compression + #_source.enabled: false + + +# ================================== General =================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. +#fields: +# env: staging + +# ================================= Dashboards ================================= +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards is disabled by default and can be enabled either by setting the +# options here or by using the `setup` command. +#setup.dashboards.enabled: false + +# The URL from where to download the dashboards archive. By default this URL +# has a value which is computed based on the Beat name and version. For released +# versions, this URL points to the dashboard archive on the artifacts.elastic.co +# website. +#setup.dashboards.url: + +# =================================== Kibana =================================== + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + #host: "localhost:5601" + + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + +# =============================== Elastic Cloud ================================ + +# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +# ================================== Outputs =================================== + +# Configure what output to use when sending the data collected by the beat. + +# ---------------------------- Elasticsearch Output ---------------------------- +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["{{ filebeat_elasticsearch_hosts | join('", "') }}"] + + # Protocol - either `http` (default) or `https`. + protocol: "{{ filebeat_elasticsearch_protocol | default('http') }}" + + # Authentication credentials - either API key or username/password. +{% if filebeat_elasticsearch_auth_api_key %} + api_key: "{{ filebeat_elasticsearch_auth_api_key }}" +{% endif %} +{% if filebeat_elasticsearch_auth_username %} + username: "{{ filebeat_elasticsearch_auth_username }}" +{% endif %} +{% if filebeat_elasticsearch_auth_password %} + password: "{{ filebeat_elasticsearch_auth_password }}" +{% endif %} + +# ------------------------------ Logstash Output ------------------------------- +#output.logstash: + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + +# ================================= Processors ================================= +processors: + - add_host_metadata: ~ +{% if filebeat_processors_cloud_metadata %} + - add_cloud_metadata: ~ +{% endif %} + - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ + +# ================================== Logging =================================== + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +#logging.level: debug + +# At debug level, you can selectively enable logging only for some components. +# To enable all selectors use ["*"]. Examples of other selectors are "beat", +# "publish", "service". +#logging.selectors: ["*"] + +# ============================= X-Pack Monitoring ============================== +# Filebeat can export internal metrics to a central Elasticsearch monitoring +# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The +# reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +#monitoring.enabled: false + +# Sets the UUID of the Elasticsearch cluster under which monitoring data for this +# Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch +# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. +#monitoring.cluster_uuid: + +# Uncomment to send the metrics to Elasticsearch. Most settings from the +# Elasticsearch output are accepted here as well. +# Note that the settings should point to your Elasticsearch *monitoring* cluster. +# Any setting that is not set is automatically inherited from the Elasticsearch +# output configuration, so if you have the Elasticsearch output configured such +# that it is pointing to your Elasticsearch monitoring cluster, you can simply +# uncomment the following line. +#monitoring.elasticsearch: + +# ============================== Instrumentation =============================== + +# Instrumentation support for the filebeat. +#instrumentation: + # Set to true to enable instrumentation of filebeat. + #enabled: false + + # Environment in which filebeat is running on (eg: staging, production, etc.) + #environment: "" + + # APM Server hosts to report instrumentation results to. + #hosts: + # - http://localhost:8200 + + # API Key for the APM Server(s). + # If api_key is set then secret_token will be ignored. + #api_key: + + # Secret token for the APM Server(s). + #secret_token: + + +# ================================= Migration ================================== + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: true From 23cd57e9aa4951d33a229990e84a2ecbdaea6485 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 18 Aug 2020 14:01:09 +0200 Subject: [PATCH 81/88] metricbeat: allow using a template --- CHANGELOG.md | 1 + metricbeat/README.md | 2 +- metricbeat/defaults/main.yml | 17 +- metricbeat/tasks/main.yml | 116 ++++++----- .../templates/metricbeat.default.yml.j2 | 180 ++++++++++++++++++ 5 files changed, 270 insertions(+), 46 deletions(-) create mode 100644 metricbeat/templates/metricbeat.default.yml.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 874df943..b1c38ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The **patch** part changes incrementally at each release. * haproxy: preconfigure SSL with defaults * lxc-php: Install php-sqlite by default * lxc-php: Don't disable putenv() by default in PHP settings +* metricbeat: allow using a template * mysql: activate binary logs by specifying log_bin path * mysql: specify a custom server_id * mysql: option to define as read only diff --git a/metricbeat/README.md b/metricbeat/README.md index 443e1107..ad348053 100644 --- a/metricbeat/README.md +++ b/metricbeat/README.md @@ -1,4 +1,4 @@ -# filebeat +# metricbeat Install Metricbeat. diff --git a/metricbeat/defaults/main.yml b/metricbeat/defaults/main.yml index df2d9649..51113271 100644 --- a/metricbeat/defaults/main.yml +++ b/metricbeat/defaults/main.yml @@ -1,10 +1,25 @@ --- elastic_stack_version: "6.x" -metricbeat_elasticsearch_protocol: "" metricbeat_elasticsearch_hosts: - "localhost:9200" +metricbeat_elasticsearch_protocol: "" +metricbeat_elasticsearch_auth_api_key: "" metricbeat_elasticsearch_auth_username: "" metricbeat_elasticsearch_auth_password: "" metricbeat_processors_cloud_metadata: False + +metricbeat_use_config_template: False +metricbeat_update_config: True +metricbeat_force_config: True + +# Example : +# metricbeat_tags: +# - "service-X" +# - "web-tier" +metricbeat_tags: Null +# Example : +# metricbeat_fields: +# - "env: staging" +metricbeat_fields: Null diff --git a/metricbeat/tasks/main.yml b/metricbeat/tasks/main.yml index 535037ec..1ae3a2cd 100644 --- a/metricbeat/tasks/main.yml +++ b/metricbeat/tasks/main.yml @@ -40,51 +40,79 @@ name: metricbeat enabled: yes -- name: Metricbeat knows where to find Elasticsearch - lineinfile: - dest: /etc/metricbeat/metricbeat.yml - regexp: '^ hosts: .*' - line: " hosts: [\"{{ metricbeat_elasticsearch_hosts | join('\", \"') }}\"]" - insertafter: "output.elasticsearch:" - notify: restart metricbeat - when: - - metricbeat_elasticsearch_hosts +# When we don't use a config template (default) +- block: + - name: Metricbeat knows where to find Elasticsearch + lineinfile: + dest: /etc/metricbeat/metricbeat.yml + regexp: '^ hosts: .*' + line: " hosts: [\"{{ metricbeat_elasticsearch_hosts | join('\", \"') }}\"]" + insertafter: "output.elasticsearch:" + notify: restart metricbeat + when: + - metricbeat_elasticsearch_hosts -- name: Metricbeat protocol for Elasticsearch - lineinfile: - dest: /etc/metricbeat/metricbeat.yml - regexp: '^ #?protocol: .*' - line: " protocol: \"{{ metricbeat_elasticsearch_protocol }}\"" - insertafter: "output.elasticsearch:" - notify: restart metricbeat - when: metricbeat_elasticsearch_protocol == "http" or metricbeat_elasticsearch_protocol == "https" + - name: Metricbeat protocol for Elasticsearch + lineinfile: + dest: /etc/metricbeat/metricbeat.yml + regexp: '^ #?protocol: .*' + line: " protocol: \"{{ metricbeat_elasticsearch_protocol }}\"" + insertafter: "output.elasticsearch:" + notify: restart metricbeat + when: metricbeat_elasticsearch_protocol == "http" or metricbeat_elasticsearch_protocol == "https" -- name: Metricbeat auth/username for Elasticsearch are configured - lineinfile: - dest: /etc/metricbeat/metricbeat.yml - regexp: '{{ item.regexp }}' - line: '{{ item.line }}' - insertafter: "output.elasticsearch:" - with_items: - - { regexp: '^ #?username: .*', line: ' username: "{{ metricbeat_elasticsearch_auth_username }}"' } - - { regexp: '^ #?password: .*', line: ' password: "{{ metricbeat_elasticsearch_auth_password }}"' } - notify: restart metricbeat - when: - - metricbeat_elasticsearch_auth_username != "" - - metricbeat_elasticsearch_auth_password != "" + - name: Metricbeat auth/username for Elasticsearch are configured + lineinfile: + dest: /etc/metricbeat/metricbeat.yml + regexp: '{{ item.regexp }}' + line: '{{ item.line }}' + insertafter: "output.elasticsearch:" + with_items: + - { regexp: '^ #?username: .*', line: ' username: "{{ metricbeat_elasticsearch_auth_username }}"' } + - { regexp: '^ #?password: .*', line: ' password: "{{ metricbeat_elasticsearch_auth_password }}"' } + notify: restart metricbeat + when: + - metricbeat_elasticsearch_auth_username + - metricbeat_elasticsearch_auth_password -- name: disable cloud_metadata - replace: - dest: /etc/metricbeat/metricbeat.yml - regexp: '^(\s+)(- add_cloud_metadata:)' - replace: '\1# \2' - notify: restart metricbeat - when: not metricbeat_processors_cloud_metadata + - name: Metricbeat api_key for Elasticsearch are configured + lineinfile: + dest: /etc/metricbeat/metricbeat.yml + regexp: '^ #?api_key: .*' + line: ' api_key: "{{ metricbeat_elasticsearch_auth_api_key }}"' + insertafter: "output.elasticsearch:" + notify: restart metricbeat + when: metricbeat_elasticsearch_auth_api_key -- name: cloud_metadata processor is disabled - lineinfile: - dest: /etc/metricbeat/metricbeat.yml - line: " - add_cloud_metadata: ~" - insert_after: '^processors:' - notify: restart metricbeat - when: metricbeat_processors_cloud_metadata + - name: disable cloud_metadata + replace: + dest: /etc/metricbeat/metricbeat.yml + regexp: '^(\s+)(- add_cloud_metadata:)' + replace: '\1# \2' + notify: restart metricbeat + when: not metricbeat_processors_cloud_metadata + + - name: cloud_metadata processor is disabled + lineinfile: + dest: /etc/metricbeat/metricbeat.yml + line: " - add_cloud_metadata: ~" + insert_after: '^processors:' + notify: restart metricbeat + when: metricbeat_processors_cloud_metadata + when: not metricbeat_use_config_template + +# When we use a config template +- block: + - name: Configuration is up-to-date + template: + src: "{{ item }}" + dest: /etc/metricbeat/metricbeat.yml + force: "{{ metricbeat_force_config }}" + with_first_found: + - "templates/metricbeat/metricbeat.{{ inventory_hostname }}.yml.j2" + - "templates/metricbeat/metricbeat.{{ host_group }}.yml.j2" + - "templates/metricbeat/metricbeat.default.yml.j2" + - "metricbeat.default.yml.j2" + notify: restart metricbeat + when: metricbeat_update_config + when: metricbeat_use_config_template diff --git a/metricbeat/templates/metricbeat.default.yml.j2 b/metricbeat/templates/metricbeat.default.yml.j2 new file mode 100644 index 00000000..bd88cdc6 --- /dev/null +++ b/metricbeat/templates/metricbeat.default.yml.j2 @@ -0,0 +1,180 @@ +###################### Metricbeat Configuration Example ####################### + +# This file is an example configuration file highlighting only the most common +# options. The metricbeat.reference.yml file from the same directory contains all the +# supported options with more comments. You can use it as a reference. +# +# You can find the full configuration reference here: +# https://www.elastic.co/guide/en/beats/metricbeat/index.html + +# =========================== Modules configuration ============================ + +metricbeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s + +# ======================= Elasticsearch template setting ======================= + +setup.template.settings: + index.number_of_shards: 1 + index.codec: best_compression + #_source.enabled: false + + +# ================================== General =================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. +{% if metricbeat_tags %} +tags: ["{{ metricbeat_tags | join('", "') }}"] +{% endif %} + +# Optional fields that you can specify to add additional information to the +# output. +{% if metricbeat_fields %} +fields: +{% for field in metricbeat_fields %} + {{ field }} +{% endfor %} +{% endif %} + +# ================================= Dashboards ================================= +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards is disabled by default and can be enabled either by setting the +# options here or by using the `setup` command. +#setup.dashboards.enabled: false + +# The URL from where to download the dashboards archive. By default this URL +# has a value which is computed based on the Beat name and version. For released +# versions, this URL points to the dashboard archive on the artifacts.elastic.co +# website. +#setup.dashboards.url: + +# =================================== Kibana =================================== + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + #host: "localhost:5601" + + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + +# =============================== Elastic Cloud ================================ + +# These settings simplify using Metricbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +# ================================== Outputs =================================== + +# Configure what output to use when sending the data collected by the beat. + +# ---------------------------- Elasticsearch Output ---------------------------- +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["{{ metricbeat_elasticsearch_hosts | join('", "') }}"] + + # Protocol - either `http` (default) or `https`. + protocol: "{{ metricbeat_elasticsearch_protocol | default('http') }}" + + # Authentication credentials - either API key or username/password. +{% if metricbeat_elasticsearch_auth_api_key %} + api_key: "{{ metricbeat_elasticsearch_auth_api_key }}" +{% endif %} +{% if metricbeat_elasticsearch_auth_username %} + username: "{{ metricbeat_elasticsearch_auth_username }}" +{% endif %} +{% if metricbeat_elasticsearch_auth_password %} + password: "{{ metricbeat_elasticsearch_auth_password }}" +{% endif %} + +# ------------------------------ Logstash Output ------------------------------- +#output.logstash: + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + +# ================================= Processors ================================= + +# Configure processors to enhance or manipulate events generated by the beat. + +processors: + - add_host_metadata: ~ +{% if metricbeat_processors_cloud_metadata %} + - add_cloud_metadata: ~ +{% endif %} + - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ + + +# ================================== Logging =================================== + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +#logging.level: debug + +# At debug level, you can selectively enable logging only for some components. +# To enable all selectors use ["*"]. Examples of other selectors are "beat", +# "publish", "service". +#logging.selectors: ["*"] + +# ============================= X-Pack Monitoring ============================== +# Metricbeat can export internal metrics to a central Elasticsearch monitoring +# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The +# reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +#monitoring.enabled: false + +# Sets the UUID of the Elasticsearch cluster under which monitoring data for this +# Metricbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch +# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. +#monitoring.cluster_uuid: + +# Uncomment to send the metrics to Elasticsearch. Most settings from the +# Elasticsearch output are accepted here as well. +# Note that the settings should point to your Elasticsearch *monitoring* cluster. +# Any setting that is not set is automatically inherited from the Elasticsearch +# output configuration, so if you have the Elasticsearch output configured such +# that it is pointing to your Elasticsearch monitoring cluster, you can simply +# uncomment the following line. +#monitoring.elasticsearch: + +# ================================= Migration ================================== + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: true From 8ab79d5ecef696f9cec6cc3713765cbdb3a00113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Thu, 20 Aug 2020 15:49:22 +0900 Subject: [PATCH 82/88] Updated CHANGELOG.md with recent merges --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1c38ed1..aaf4836e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ The **patch** part changes incrementally at each release. ### Added +* generate-ldif: add NVMe disk support +* nagios-nrpe/evolinux-base: brand new check for hardware raid on HP servers gen 10 * certbot: detect HAProxy cert directory * filebeat: allow using a template * haproxy: add deny_ips file to reject connections @@ -52,6 +54,10 @@ The **patch** part changes incrementally at each release. ### Fixed +* squid: better regex to match sa-update domains +* nagios-nrpe: check_amavis: updated regex +* generate-ldif: better detection of computerOS field +* generate-ldif: skip some odd ethernet devices * certbot: restore compatibility with old Nginx * lxc-php: Install opensmtpd as intended * mongodb: fix logrotate patterm on Debian buster From bf0676cbf88a9b34eb7be8df662a54047716e28a Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 21 Aug 2020 13:36:24 +0200 Subject: [PATCH 83/88] evoacme: update for new certbot role * certbot is installed by the certbot role * Apache/Nginx configuration is delegated to the certbot role * No more "acme" user, everything is done with "root". --- CHANGELOG.md | 1 + evoacme/tasks/acme.yml | 61 ---------------------------------- evoacme/tasks/apache.yml | 25 -------------- evoacme/tasks/certbot.yml | 33 +++--------------- evoacme/tasks/evoacme_hook.yml | 5 +++ evoacme/tasks/main.yml | 20 +---------- evoacme/tasks/nginx.yml | 35 ------------------- evoacme/tasks/permissions.yml | 33 ++++++++++++++++++ 8 files changed, 44 insertions(+), 169 deletions(-) delete mode 100644 evoacme/tasks/acme.yml delete mode 100644 evoacme/tasks/apache.yml delete mode 100644 evoacme/tasks/nginx.yml create mode 100644 evoacme/tasks/permissions.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index aaf4836e..bf5d40eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ The **patch** part changes incrementally at each release. * elasticsearch: configure cluster with seed hosts and initial masters * evoacme: upstream release 20.06.1 * evoacme: read values from environment before defaults file +* evoacme: update for new certbot role * haproxy: deport SSL tuning to Mozilla SSL generator * haproxy: chroot and socket path are configurable * haproxy: adapt backports installed package list to distibution diff --git a/evoacme/tasks/acme.yml b/evoacme/tasks/acme.yml deleted file mode 100644 index 16417ca6..00000000 --- a/evoacme/tasks/acme.yml +++ /dev/null @@ -1,61 +0,0 @@ ---- -- name: Create acme group - group: - name: acme - state: present - -- name: Create acme user - user: - name: acme - group: acme - state: present - createhome: no - home: "{{ evoacme_acme_dir }}" - shell: /bin/false - system: yes - -- name: Fix crt dir's right - file: - path: "{{ evoacme_crt_dir }}" - mode: "0755" - owner: acme - group: acme - state: directory - -- name: "Fix hooks directory permissions" - file: - path: "{{ evoacme_hooks_dir }}" - mode: "0700" - owner: acme - group: acme - state: directory - -- name: Fix log dir's right - file: - path: "{{ evoacme_log_dir }}" - mode: "0755" - owner: acme - group: acme - state: directory - -- name: Fix challenge dir's right - file: - path: "{{ evoacme_acme_dir }}" - mode: "0755" - owner: acme - group: acme - state: directory - -- name: Is /etc/aliases present? - stat: - path: /etc/aliases - register: etc_aliases - -- name: Set acme aliases - lineinfile: - state: present - dest: /etc/aliases - line: 'acme: root' - regexp: 'acme:' - when: etc_aliases.stat.exists - notify: "newaliases" diff --git a/evoacme/tasks/apache.yml b/evoacme/tasks/apache.yml deleted file mode 100644 index f12aee32..00000000 --- a/evoacme/tasks/apache.yml +++ /dev/null @@ -1,25 +0,0 @@ -- name: Create conf dirs - file: - path: "/etc/apache2/{{ item }}" - state: directory - with_items: - - 'conf-available' - - 'conf-enabled' - -- name: Copy acme challenge conf - template: - src: templates/apache.conf.j2 - dest: /etc/apache2/conf-available/letsencrypt.conf - owner: root - group: root - mode: "0644" - notify: reload apache2 - -- name: Enable acme challenge conf - file: - src: /etc/apache2/conf-available/letsencrypt.conf - dest: /etc/apache2/conf-enabled/letsencrypt.conf - state: link - owner: root - group: root - notify: reload apache2 diff --git a/evoacme/tasks/certbot.yml b/evoacme/tasks/certbot.yml index 5e18cfec..0577abbe 100644 --- a/evoacme/tasks/certbot.yml +++ b/evoacme/tasks/certbot.yml @@ -1,45 +1,20 @@ --- - -- name: Use backports for jessie - block: - - name: install jessie-backports - include_role: - name: evolix/apt - tasks_from: backports.yml - - - name: Add exceptions for certbot dependencies - copy: - src: backports-certbot - dest: /etc/apt/preferences.d/z-backports-certbot - notify: apt update - - - meta: flush_handlers - when: ansible_distribution_release == "jessie" - -- name: Install certbot with apt - apt: - name: certbot - state: latest +- include_role: + name: evolix/certbot - include_role: name: evolix/remount-usr -- name: Remove certbot symlink for apt install - file: - path: /usr/local/bin/certbot - state: absent - name: Disable /etc/cron.d/certbot - command: mv /etc/cron.d/certbot /etc/cron.d/certbot.disabled + command: mv -f /etc/cron.d/certbot /etc/cron.d/certbot.disabled args: removes: /etc/cron.d/certbot - creates: /etc/cron.d/certbot.disabled - name: Disable /etc/cron.daily/certbot - command: mv /etc/cron.daily/certbot /etc/cron.daily/certbot.disabled + command: mv -f /etc/cron.daily/certbot /etc/cron.daily/certbot.disabled args: removes: /etc/cron.daily/certbot - creates: /etc/cron.daily/certbot.disabled - name: Install evoacme custom cron copy: diff --git a/evoacme/tasks/evoacme_hook.yml b/evoacme/tasks/evoacme_hook.yml index 200bcbc4..51dbb21c 100644 --- a/evoacme/tasks/evoacme_hook.yml +++ b/evoacme/tasks/evoacme_hook.yml @@ -1,5 +1,10 @@ --- +- name: "Create {{ hook_name }} hook directory" + file: + dest: "{{ evoacme_hooks_dir }}" + state: directory + - name: "Search for {{ hook_name }} hook" command: "find {{ evoacme_hooks_dir }} -type f \\( -name '{{ hook_name }}' -o -name '{{ hook_name }}.*' \\)" check_mode: no diff --git a/evoacme/tasks/main.yml b/evoacme/tasks/main.yml index ac20cabb..1baede00 100644 --- a/evoacme/tasks/main.yml +++ b/evoacme/tasks/main.yml @@ -7,7 +7,7 @@ - include: certbot.yml -- include: acme.yml +- include: permissions.yml - include: evoacme_hook.yml vars: @@ -22,21 +22,3 @@ - include: conf.yml - include: scripts.yml - -- name: Determine Apache presence - stat: - path: /etc/apache2/apache2.conf - check_mode: no - register: sta - -- name: Determine Nginx presence - stat: - path: /etc/nginx/nginx.conf - check_mode: no - register: stn - -- include: apache.yml - when: sta.stat.isreg is defined and sta.stat.isreg - -- include: nginx.yml - when: stn.stat.isreg is defined and stn.stat.isreg diff --git a/evoacme/tasks/nginx.yml b/evoacme/tasks/nginx.yml deleted file mode 100644 index 44fc6656..00000000 --- a/evoacme/tasks/nginx.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- - -- name: move acme challenge conf if missplaced - command: mv /etc/nginx/letsencrypt.conf /etc/nginx/snippets/letsencrypt.conf - args: - removes: /etc/nginx/letsencrypt.conf - creates: /etc/nginx/snippets/letsencrypt.conf - -- name: Copy acme challenge conf - template: - src: templates/nginx.conf.j2 - dest: /etc/nginx/snippets/letsencrypt.conf - owner: root - group: root - mode: "0644" - -- name: look for old path - command: grep -r /etc/nginx/letsencrypt.conf /etc/nginx - changed_when: False - failed_when: False - check_mode: no - register: grep_letsencrypt_old_path - -- name: Keep a symlink for vhosts with old path - file: - src: /etc/nginx/snippets/letsencrypt.conf - dest: /etc/nginx/letsencrypt.conf - state: link - when: grep_letsencrypt_old_path.rc == 0 - -- name: Remove symlink if no vhost with old path - file: - dest: /etc/nginx/letsencrypt.conf - state: absent - when: grep_letsencrypt_old_path.rc == 1 diff --git a/evoacme/tasks/permissions.yml b/evoacme/tasks/permissions.yml new file mode 100644 index 00000000..69bcbe12 --- /dev/null +++ b/evoacme/tasks/permissions.yml @@ -0,0 +1,33 @@ +--- + +- name: Fix crt directory permissions + file: + path: "{{ evoacme_crt_dir }}" + mode: "0755" + owner: root + group: root + state: directory + +- name: "Fix hooks directory permissions" + file: + path: "{{ evoacme_hooks_dir }}" + mode: "0700" + owner: root + group: root + state: directory + +- name: Fix log directory permissions + file: + path: "{{ evoacme_log_dir }}" + mode: "0755" + owner: root + group: root + state: directory + +- name: Fix challenge directory permissions + file: + path: "{{ evoacme_acme_dir }}" + mode: "0755" + owner: root + group: root + state: directory From e5c759c7f09e06d9979a60acb3a44888596b9972 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 21 Aug 2020 14:01:06 +0200 Subject: [PATCH 84/88] evoacme: upstream release 20.08 --- CHANGELOG.md | 2 +- evoacme/files/evoacme.sh | 6 +++--- evoacme/files/make-csr.sh | 2 +- evoacme/files/vhost-domains.sh | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf5d40eb..15ab9284 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,7 @@ The **patch** part changes incrementally at each release. * certbot: install certbot dependencies non-interactively for jessie * elasticsearch: set tmpdir before datadir * elasticsearch: configure cluster with seed hosts and initial masters -* evoacme: upstream release 20.06.1 +* evoacme: upstream release 20.08 * evoacme: read values from environment before defaults file * evoacme: update for new certbot role * haproxy: deport SSL tuning to Mozilla SSL generator diff --git a/evoacme/files/evoacme.sh b/evoacme/files/evoacme.sh index d98f061e..e8330748 100755 --- a/evoacme/files/evoacme.sh +++ b/evoacme/files/evoacme.sh @@ -285,7 +285,7 @@ main() { export EVOACME_FULLCHAIN="${LIVE_FULLCHAIN}" # search for files in hooks directory - for hook in $(find ${HOOKS_DIR} -type f); do + for hook in $(find ${HOOKS_DIR} -type f -executable | sort); do # keep only executables files, not containing a "." if [ -x "${hook}" ] && (basename "${hook}" | grep -vqF "."); then debug "Executing ${hook}" @@ -303,7 +303,7 @@ readonly QUIET=${QUIET:-"0"} readonly TEST=${TEST:-"0"} readonly DRY_RUN=${DRY_RUN:-"0"} -readonly VERSION="20.06.1" +readonly VERSION="20.08" # Read configuration file, if it exists [ -r /etc/default/evoacme ] && . /etc/default/evoacme @@ -314,7 +314,7 @@ readonly ACME_DIR=${ACME_DIR:-"/var/lib/letsencrypt"} readonly CSR_DIR=${CSR_DIR:-"/etc/ssl/requests"} readonly CRT_DIR=${CRT_DIR:-"/etc/letsencrypt"} readonly LOG_DIR=${LOG_DIR:-"/var/log/evoacme"} -readonly HOOKS_DIR=${HOOKS_DIR:-"${CRT_DIR}/hooks"} +readonly HOOKS_DIR=${HOOKS_DIR:-"${CRT_DIR}/renewal-hooks/deploy"} readonly SSL_MINDAY=${SSL_MINDAY:-"30"} readonly SSL_EMAIL=${SSL_EMAIL:-""} diff --git a/evoacme/files/make-csr.sh b/evoacme/files/make-csr.sh index 7e08cfe0..372c58fc 100755 --- a/evoacme/files/make-csr.sh +++ b/evoacme/files/make-csr.sh @@ -265,7 +265,7 @@ readonly ARGS=$@ readonly VERBOSE=${VERBOSE:-"0"} readonly QUIET=${QUIET:-"0"} -readonly VERSION="20.06.1" +readonly VERSION="20.08" # Read configuration file, if it exists [ -r /etc/default/evoacme ] && . /etc/default/evoacme diff --git a/evoacme/files/vhost-domains.sh b/evoacme/files/vhost-domains.sh index 8b9ef3a5..41b065b6 100755 --- a/evoacme/files/vhost-domains.sh +++ b/evoacme/files/vhost-domains.sh @@ -170,7 +170,7 @@ readonly ARGS=$@ readonly VERBOSE=${VERBOSE:-"0"} readonly QUIET=${QUIET:-"0"} -readonly VERSION="20.06.1" +readonly VERSION="20.08" readonly SRV_IP=${SRV_IP:-""} From ed45551a0c781fc66dc5c4e697ada6f6ee1c19d9 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 21 Aug 2020 14:02:07 +0200 Subject: [PATCH 85/88] evoacme: use Let's Encrypt deploy hooks instead of evoacme hooks --- evoacme/defaults/main.yml | 2 +- evoacme/files/hooks/commit | 18 --------------- evoacme/files/hooks/reload_apache | 30 ------------------------- evoacme/files/hooks/reload_dovecot | 35 ------------------------------ evoacme/files/hooks/reload_nginx | 30 ------------------------- evoacme/files/hooks/reload_postfix | 35 ------------------------------ evoacme/tasks/main.yml | 7 +----- evoacme/templates/evoacme.conf.j2 | 1 + 8 files changed, 3 insertions(+), 155 deletions(-) delete mode 100644 evoacme/files/hooks/commit delete mode 100755 evoacme/files/hooks/reload_apache delete mode 100755 evoacme/files/hooks/reload_dovecot delete mode 100755 evoacme/files/hooks/reload_nginx delete mode 100755 evoacme/files/hooks/reload_postfix diff --git a/evoacme/defaults/main.yml b/evoacme/defaults/main.yml index 3d0842bc..e54ef2fc 100644 --- a/evoacme/defaults/main.yml +++ b/evoacme/defaults/main.yml @@ -5,7 +5,7 @@ evoacme_dhparam_size: 2048 evoacme_acme_dir: /var/lib/letsencrypt evoacme_csr_dir: /etc/ssl/requests evoacme_crt_dir: /etc/letsencrypt -evoacme_hooks_dir: "{{ evoacme_crt_dir }}/hooks" +evoacme_hooks_dir: "{{ evoacme_crt_dir }}/renewal-hooks/deploy" evoacme_log_dir: /var/log/evoacme evoacme_ssl_minday: 30 evoacme_ssl_ct: 'FR' diff --git a/evoacme/files/hooks/commit b/evoacme/files/hooks/commit deleted file mode 100644 index 86381dce..00000000 --- a/evoacme/files/hooks/commit +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -git_bin=$(command -v git) -letsencrypt_dir=/etc/letsencrypt -export GIT_DIR="/etc/.git" -export GIT_WORK_TREE="/etc" - -if test -x "${git_bin}" && test -d "${GIT_DIR}" && test -d "${GIT_WORK_TREE}"; then - changed_lines=$(${git_bin} status --porcelain -- ${letsencrypt_dir} | wc -l | tr -d ' ') - - if [ "${changed_lines}" != "0" ]; then - ${git_bin} add --all ${letsencrypt_dir} - message="[letsencrypt] certificates renewal (${RENEWED_DOMAINS})" - ${git_bin} commit --message "${message}" --quiet - else - echo "Weird, nothing has changed but the hook has been executed for '${RENEWED_DOMAINS}'" - fi -fi diff --git a/evoacme/files/hooks/reload_apache b/evoacme/files/hooks/reload_apache deleted file mode 100755 index 1db59b5b..00000000 --- a/evoacme/files/hooks/reload_apache +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh - -readonly PROGNAME=$(basename "$0") -# shellcheck disable=SC2124,SC2034 -readonly ARGS=$@ - -readonly VERBOSE=${VERBOSE:-"0"} -readonly QUIET=${QUIET:-"0"} - -error() { - >&2 echo "${PROGNAME}: $1" - exit 1 -} -debug() { - if [ "${VERBOSE}" = "1" ] && [ "${QUIET}" != "1" ]; then - >&2 echo "${PROGNAME}: $1" - fi -} - -if [ -n "$(pidof apache2)" ]; then - # shellcheck disable=SC2091 - if $($(command -v apache2ctl) -t 2> /dev/null); then - debug "Apache detected... reloading" - service apache2 reload - else - error " Apache config is broken, you must fix it !" - fi -else - debug "Apache is not running. Skip." -fi diff --git a/evoacme/files/hooks/reload_dovecot b/evoacme/files/hooks/reload_dovecot deleted file mode 100755 index 602e7c85..00000000 --- a/evoacme/files/hooks/reload_dovecot +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh - -readonly PROGNAME=$(basename "$0") -# shellcheck disable=SC2124,SC2034 -readonly ARGS=$@ - -readonly VERBOSE=${VERBOSE:-"0"} -readonly QUIET=${QUIET:-"0"} - -error() { - >&2 echo "${PROGNAME}: $1" - exit 1 -} -debug() { - if [ "${VERBOSE}" = "1" ] && [ "${QUIET}" != "1" ]; then - >&2 echo "${PROGNAME}: $1" - fi -} - -if [ -n "$(pidof dovecot)" ]; then - # shellcheck disable=SC2091 - if $($(command -v doveconf) > /dev/null); then - # shellcheck disable=SC2091 - if $($(command -v doveconf)|grep -E "^ssl_cert[^_]"|grep -q "letsencrypt"); then - debug "Dovecot detected... reloading" - service dovecot reload - else - debug "Dovecot doesn't use Let's Encrypt certificate. Skip." - fi - else - error "Dovecot config is broken, you must fix it !" - fi -else - debug "Dovecot is not running. Skip." -fi diff --git a/evoacme/files/hooks/reload_nginx b/evoacme/files/hooks/reload_nginx deleted file mode 100755 index 48c49da9..00000000 --- a/evoacme/files/hooks/reload_nginx +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh - -readonly PROGNAME=$(basename "$0") -# shellcheck disable=SC2124,SC2034 -readonly ARGS=$@ - -readonly VERBOSE=${VERBOSE:-"0"} -readonly QUIET=${QUIET:-"0"} - -error() { - >&2 echo "${PROGNAME}: $1" - exit 1 -} -debug() { - if [ "${VERBOSE}" = "1" ] && [ "${QUIET}" != "1" ]; then - >&2 echo "${PROGNAME}: $1" - fi -} - -if [ -n "$(pidof nginx)" ]; then - # shellcheck disable=SC2091 - if $($(command -v nginx) -t 2> /dev/null); then - debug "Nginx detected... reloading" - service nginx reload - else - error "Nginx config is broken, you must fix it !" - fi -else - debug "Nginx is not running. Skip." -fi diff --git a/evoacme/files/hooks/reload_postfix b/evoacme/files/hooks/reload_postfix deleted file mode 100755 index eb17b58e..00000000 --- a/evoacme/files/hooks/reload_postfix +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh - -readonly PROGNAME=$(basename "$0") -# shellcheck disable=SC2124,SC2034 -readonly ARGS=$@ - -readonly VERBOSE=${VERBOSE:-"0"} -readonly QUIET=${QUIET:-"0"} - -error() { - >&2 echo "${PROGNAME}: $1" - exit 1 -} -debug() { - if [ "${VERBOSE}" = "1" ] && [ "${QUIET}" != "1" ]; then - >&2 echo "${PROGNAME}: $1" - fi -} - -if [ -n "$(pidof master)" ]; then - # shellcheck disable=SC2091 - if $($(command -v postconf) > /dev/null); then - # shellcheck disable=SC2091 - if $($(command -v postconf)|grep -E "^smtpd_tls_cert_file"|grep -q "letsencrypt"); then - debug "Postfix detected... reloading" - service postfix reload - else - debug "Postfix doesn't use Let's Encrypt certificate. Skip." - fi - else - error "Postfix config is broken, you must fix it !" - fi -else - debug "Postfix is not running. Skip." -fi diff --git a/evoacme/tasks/main.yml b/evoacme/tasks/main.yml index 1baede00..cd7ccc70 100644 --- a/evoacme/tasks/main.yml +++ b/evoacme/tasks/main.yml @@ -12,12 +12,7 @@ - include: evoacme_hook.yml vars: hook_name: "{{ item }}" - with_items: - - reload_apache - - reload_nginx - - reload_dovecot - - reload_postfix - - commit + loop: [] - include: conf.yml diff --git a/evoacme/templates/evoacme.conf.j2 b/evoacme/templates/evoacme.conf.j2 index fc1ba70b..27405f56 100644 --- a/evoacme/templates/evoacme.conf.j2 +++ b/evoacme/templates/evoacme.conf.j2 @@ -4,5 +4,6 @@ SSL_KEY_DIR=${SSL_KEY_DIR:-{{ evoacme_ssl_key_dir }}} ACME_DIR=${ACME_DIR:-{{ evoacme_acme_dir }}} CSR_DIR=${CSR_DIR:-{{ evoacme_csr_dir }}} CRT_DIR=${CRT_DIR:-{{ evoacme_crt_dir }}} +HOOKS_DIR=${HOOKS_DIR:-"{{ evoacme_hooks_dir }}"} LOG_DIR=${LOG_DIR:-{{ evoacme_log_dir }}} SSL_MINDAY=${SSL_MINDAY:-{{ evoacme_ssl_minday }}} From 82bd1ed49a00bb68dd62b2b07eb88f8053956065 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 21 Aug 2020 14:03:41 +0200 Subject: [PATCH 86/88] sort lines in CHANGELOG --- CHANGELOG.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15ab9284..77f2ac14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,39 +12,39 @@ The **patch** part changes incrementally at each release. ### Added -* generate-ldif: add NVMe disk support -* nagios-nrpe/evolinux-base: brand new check for hardware raid on HP servers gen 10 * certbot: detect HAProxy cert directory * filebeat: allow using a template +* generate-ldif: add NVMe disk support * haproxy: add deny_ips file to reject connections * haproxy: add some comments to default config * haproxy: enable stats frontend with access lists * haproxy: preconfigure SSL with defaults -* lxc-php: Install php-sqlite by default * lxc-php: Don't disable putenv() by default in PHP settings +* lxc-php: Install php-sqlite by default * metricbeat: allow using a template * mysql: activate binary logs by specifying log_bin path -* mysql: specify a custom server_id * mysql: option to define as read only +* mysql: specify a custom server_id +* nagios-nrpe/evolinux-base: brand new check for hardware raid on HP servers gen 10 * nginx: make default vhost configurable * packweb-apache: Install zip & unzip by default -* php: Install php-sqlite by default * php: Don't disable putenv() by default in PHP settings +* php: Install php-sqlite by default ### Changed * certbot: fix haproxy hook (ssl cert directory detection) * certbot: install certbot dependencies non-interactively for jessie -* elasticsearch: set tmpdir before datadir * elasticsearch: configure cluster with seed hosts and initial masters -* evoacme: upstream release 20.08 +* elasticsearch: set tmpdir before datadir * evoacme: read values from environment before defaults file * evoacme: update for new certbot role -* haproxy: deport SSL tuning to Mozilla SSL generator -* haproxy: chroot and socket path are configurable +* evoacme: upstream release 20.08 * haproxy: adapt backports installed package list to distibution -* haproxy: split stats variables +* haproxy: chroot and socket path are configurable +* haproxy: deport SSL tuning to Mozilla SSL generator * haproxy: rotate logs with date extension and immediate compression +* haproxy: split stats variables * lxc-php: Do --no-install-recommends for ssmtp/opensmtpd * mongodb: install custom munin plugins * nginx: read server-status values before changing the config @@ -55,14 +55,14 @@ The **patch** part changes incrementally at each release. ### Fixed -* squid: better regex to match sa-update domains -* nagios-nrpe: check_amavis: updated regex +* certbot: restore compatibility with old Nginx +* evobackup-client: fixed the ssh connection test * generate-ldif: better detection of computerOS field * generate-ldif: skip some odd ethernet devices -* certbot: restore compatibility with old Nginx * lxc-php: Install opensmtpd as intended * mongodb: fix logrotate patterm on Debian buster -* evobackup-client: fixed the ssh connection test +* nagios-nrpe: check_amavis: updated regex +* squid: better regex to match sa-update domains * varnish: fix start command when multiple addresses are present ### Removed From 24dd420aaceeba8a736ef3cbf3e569b8c02c2807 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 21 Aug 2020 14:21:28 +0200 Subject: [PATCH 87/88] evoacme: disable empty task for hooks --- evoacme/tasks/main.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/evoacme/tasks/main.yml b/evoacme/tasks/main.yml index cd7ccc70..c7481ba0 100644 --- a/evoacme/tasks/main.yml +++ b/evoacme/tasks/main.yml @@ -9,10 +9,11 @@ - include: permissions.yml -- include: evoacme_hook.yml - vars: - hook_name: "{{ item }}" - loop: [] +# Enable this task if you want to deploy hooks +# - include: evoacme_hook.yml +# vars: +# hook_name: "{{ item }}" +# loop: [] - include: conf.yml From fce6676eca053a4efed9b69577010b3f4eea200f Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 21 Aug 2020 14:50:17 +0200 Subject: [PATCH 88/88] Release 10.1.0 --- CHANGELOG.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77f2ac14..d58532af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,18 @@ The **patch** part changes incrementally at each release. ### Added +### Changed + +### Fixed + +### Removed + +### Security + +## [10.1.0] 2020-08-21 + +### Added + * certbot: detect HAProxy cert directory * filebeat: allow using a template * generate-ldif: add NVMe disk support @@ -65,10 +77,6 @@ The **patch** part changes incrementally at each release. * squid: better regex to match sa-update domains * varnish: fix start command when multiple addresses are present -### Removed - -### Security - ## [10.0.0] - 2020-05-13 ### Added