accounts: create only users who have a certain value for the create key (default: always)

CHANGELOG

@@ -24,11 +24,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * nagios-nrpe: add some information in check_connections_state.sh check
 * ospf: precise in the readme file that no daemon is configured/activated
 * logsentry: delete unused default file that we put in /usr/share/scripts
+* base: set the lookup option so that resolv.conf searches /etc/hosts before querying a domain name server; the default is the opposite
+* post-install: add the pf_states check by default in generateldif.sh script
+* nagios-nrpe: allow older cipher suites for older Icinga version
+* evobackup: execute canary script before executing backup script
+* accounts: create only users who have a certain value for the `create` key (default: `always`)
 
 ### Changed
 
 * accounts: use "evobsd_internal_group" for SSH authentication
-* evocheck: imported version 22.03
 * base: zzz_evobackup upstream release 22.03
 * etc-git: manage commits with an optimized shell script instead of many slow Ansible tasks
 * etc-git: use "ansible-commit" to efficiently commit all available repositories from Ansible
@@ -61,6 +65,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * collectd: improve dns_stats.sh script for more metrics
 * ospf: do not repeat use of command, use variable instead with output of command
 * nagios-nrpe: changed check_load to make it more relevant
+* nagios-nrpe: check_ipsecctl.sh is never used standalone for check_vpn, always called by check_ipsecctl_critiques.sh
+* evobackup: zzz_evobackup upstream release 22.12, and call zzz_evobackup with bash
+* base: install bash, now needed for zzz_evobackup script
+* post-install: execute motd-carp-state.sh only once an hour
+* collectd: modified collectd scripts directory and scripts files right so that only _collectd group can execute them
+* base: install ncdu and htop often used as diagnostic tools
+* base: dump-server-state.sh upstream release 23.06
+* evocheck: upstream release 23.06
+* base: add evobsd_alias_fwupdate variable and make kshrc file a template so we can set or not a fw_update alias to servers that need it
 
 ### Fixed
 
@@ -71,6 +84,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * nagios-nrpe: grep in check_ipsecctl_critiques.sh was too large
 * post-install: fix missing space in generateldif.sh script
 * logsentry: fix variables for configuration files
+* nagios-nrpe: fix allowed_hosts configuration: keep potential added IP, but we cannot use backrefs if the line does not exist yet
+* accounts: configure user home, ssh keys and groups only if it already exists, so that there is no error when run in check mode and user doesn't exist yet
 
 ### Removed
 

roles/accounts/defaults/main.yml

@@ -1,2 +1,5 @@
 ---
 evobsd_root_login: "no"
+
+# Defines which groups of users are created
+evobsd_users_create: "always"

roles/accounts/tasks/main.yml

@@ -17,7 +17,9 @@
   vars:
     user: "{{ item.value }}"
   with_dict: "{{ evolix_users }}"
-  when: evolix_users != {}
+  when: 
+    - user.create == evobsd_users_create
+    - evolix_users != {}
   tags:
     - accounts
     - admin

roles/accounts/tasks/user.yml

@@ -23,6 +23,10 @@
     - accounts
     - admin
 
+- name: "Gather available local users"
+  getent:
+    database: passwd
+
 - name: "Home directory for '{{ user.name }}' is only accesible by owner"
   file:
     name: '/home/{{ user.name }}'
@@ -30,6 +34,7 @@
     owner: "{{ user.name }}"
     group: "{{ user.name }}"
     state: directory
+  when: user.name in getent_passwd.keys()
   tags:
     - accounts
     - admin
@@ -42,7 +47,9 @@
   with_items: "{{ user.ssh_keys }}"
   loop_control:
     loop_var: ssk_key
-  when: user.ssh_keys is defined
+  when:
+    - user.ssh_keys is defined
+    - user.name in getent_passwd.keys()
   tags:
     - accounts
     - admin
@@ -58,6 +65,7 @@
     - "{{ evobsd_sudo_group }}"
   loop_control:
     loop_var: groups_item
+  when: user.name in getent_passwd.keys()
   tags:
     - accounts
     - admin

roles/base/defaults/main.yml

@@ -4,6 +4,7 @@
 # dotfiles.yml
 evobsd_system_timeout: 36000
 evobsd_path: "$HOME/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
+evobsd_alias_fwupdate: false
 
 # mail.yml
 general_alert_email: "root@localhost"

roles/base/files/dump-server-state.sh

@@ -3,7 +3,7 @@
 PROGNAME="dump-server-state"
 REPOSITORY="https://gitea.evolix.org/evolix/dump-server-state"
 
-VERSION="22.03"
+VERSION="23.06"
 readonly VERSION
 
 dump_dir=
@@ -293,7 +293,7 @@ task_disks() {
         for disk in ${disks}; do
             disklabel_bin=$(command -v disklabel)
             if [ -n "${disklabel_bin}" ]; then
-                last_result=$(${disklabel_bin} "${disk}" 2>/dev/null > "${dump_dir}/partitions-${disk}")
+                last_result=$(${disklabel_bin} "${disk}" 2>&1 > "${dump_dir}/partitions-${disk}")
                 last_rc=$?
 
                 if [ ${last_rc} -eq 0 ]; then
@@ -309,10 +309,10 @@ task_disks() {
 
             bioctl_bin=$(command -v bioctl)
             if [ -n "${bioctl_bin}" ]; then
-                last_result=$(${bioctl_bin} "${disk}" 2>/dev/null > "${dump_dir}/bioctl-${disk}")
+                last_result=$(${bioctl_bin} "${disk}" 2>&1 > "${dump_dir}/bioctl-${disk}")
                 last_rc=$?
 
-                if [ ${last_rc} -eq 0 ]; then
+                if [ ${last_rc} -eq 0 ] || { [ ${last_rc} -ne 0 ] && [ "${last_result}" = "bioctl: DIOCINQ: Inappropriate ioctl for device" ]; }; then
                     debug "* bioctl ${disk} OK"
                 else
                     debug "* bioctl ${disk} ERROR"
@@ -325,10 +325,10 @@ task_disks() {
 
             atactl_bin=$(command -v atactl)
             if [ -n "${atactl_bin}" ]; then
-                last_result=$(${atactl_bin} "${disk}" 2>/dev/null > "${dump_dir}/atactl-${disk}")
+                last_result=$(${atactl_bin} "${disk}" 2>&1 > "${dump_dir}/atactl-${disk}")
                 last_rc=$?
 
-                if [ ${last_rc} -eq 0 ]; then
+                if [ ${last_rc} -eq 0 ] || { [ ${last_rc} -ne 0 ] && [ "${last_result}" = "atactl: ATA device returned error register 0" ]; }; then
                     debug "* atactl ${disk} OK"
                 else
                     debug "* atactl ${disk} ERROR"
@@ -432,7 +432,7 @@ task_rcctl() {
         last_result=$(${rcctl_bin} ls failed > "${dump_dir}/rcctl-failed-services.txt")
         last_rc=$?
 
-        if [ ${last_rc} -eq 0 ]; then
+        if [ ${last_rc} -eq 0 ] || [ ${last_rc} -eq 1 ]; then
             debug "* failed services OK"
         else
             debug "* failed services ERROR"

roles/base/tasks/dotfiles.yml

@@ -18,8 +18,8 @@
     - dotfiles
 
 - name: "Customize .kshrc environment file"
-  copy:
-    src: kshrc
+  template:
+    src: kshrc.j2
     dest: /root/.kshrc
     mode: "0644"
   tags:
@@ -57,8 +57,8 @@
     - dotfiles
 
 - name: "Customize .kshrc environment file for new users"
-  copy:
-    src: kshrc
+  template:
+    src: kshrc.j2
     dest: /etc/skel/.kshrc
     mode: "0644"
   tags:

roles/base/tasks/main.yml

@@ -1,5 +1,6 @@
 ---
 # tasks file for evobsd-base
+- include: resolvconf.yml
 - include: packages.yml
 - include: doas.yml
 - include: dotfiles.yml

roles/base/tasks/packages.yml

@@ -17,6 +17,9 @@
       - mtr--
       - iftop
       - sudo--
+      - bash
+      - ncdu
+      - htop
   tags:
     - pkg
 

roles/base/tasks/resolvconf.yml

@@ -0,0 +1,11 @@
+---
+- name: "Configure resolv.conf"
+  lineinfile:
+    path: /etc/resolv.conf
+    line: "lookup file bind"
+    insertbefore: BOF
+    owner: root
+    group: wheel
+    mode: "0644"
+  tags:
+    - resolvconf

roles/base/tasks/sudo.yml

@@ -8,7 +8,7 @@
   tags:
     - sudo
 
-- name: "Configure sudoers for evomaintenance and monitoring"
+- name: "Configure sudoers"
   blockinfile:
     state: present
     dest: /etc/sudoers

roles/base/templates/kshrc.j2

@@ -1,3 +1,6 @@
+{% if evobsd_alias_fwupdate %}
+alias fw_update="{{ evobsd_alias_fwupdate }}"
+{% endif %}
 alias vi='vim'
 sudo() { if [[ $# == "1" ]] && [[ $1 == "su" ]]; then command sudo -i; else command sudo "$@"; fi }
 alias history="fc -l 0"

roles/collectd/tasks/main.yml

@@ -26,6 +26,9 @@
   file:
     path: /usr/local/share/collectd/scripts
     state: directory
+    owner: "root"
+    group: "_collectd"
+    mode: "0710"
   when: collectd_plugin_exec_ifq_drops or collectd_plugin_exec_dns_stats or collectd_plugin_exec_dhcp_pool
   tags:
     - collectd
@@ -34,7 +37,9 @@
   copy:
     src: ifq_drops.sh
     dest: /usr/local/share/collectd/scripts/ifq_drops.sh
-    mode: 0755
+    owner: "root"
+    group: "_collectd"
+    mode: "0710"
     force: true
   when: collectd_plugin_exec_ifq_drops
   tags:
@@ -52,7 +57,9 @@
   copy:
     src: dns_stats.sh
     dest: /usr/local/share/collectd/scripts/dns_stats.sh
-    mode: 0755
+    owner: "root"
+    group: "_collectd"
+    mode: "0710"
     force: true
   when: collectd_plugin_exec_dns_stats
   tags:
@@ -78,7 +85,9 @@
   copy:
     src: dhcp_pool.pl
     dest: /usr/local/share/collectd/scripts/dhcp_pool.pl
-    mode: 0755
+    owner: "root"
+    group: "_collectd"
+    mode: "0710"
     force: true
   when: collectd_plugin_exec_dhcp_pool
   tags:

roles/evobackup/files/000-update-evobackup-canary

@@ -0,0 +1,3 @@
+#!/bin/sh
+                      
+update-evobackup-canary --who @daily

roles/evobackup/files/zzz_evobackup

@@ -1,26 +1,24 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Script Evobackup client
 # See https://gitea.evolix.org/evolix/evobackup
 #
-# Author: Gregory Colpart <reg@evolix.fr>
-# Contributors:
-# Romain Dessort <rdessort@evolix.fr>
-# Benoît Série <bserie@evolix.fr>
-# Tristan Pilat <tpilat@evolix.fr>
-# Victor Laborie <vlaborie@evolix.fr>
-# Jérémy Lecour <jlecour@evolix.fr>
+# Authors: Evolix <info@evolix.fr>,
+#          Gregory Colpart <reg@evolix.fr>,
+#          Romain Dessort <rdessort@evolix.fr>,
+#          Benoit Série <bserie@evolix.fr>,
+#          Tristan Pilat <tpilat@evolix.fr>,
+#          Victor Laborie <vlaborie@evolix.fr>,
+#          Jérémy Lecour <jlecour@evolix.fr>
+#          and others.
 #
 # Licence: AGPLv3
 #
 # /!\ DON'T FORGET TO SET "MAIL" and "SERVERS" VARIABLES
 
-# Fail on unassigned variables
-set -u
-
 ##### Configuration ###################################################
 
-VERSION="22.03"
+VERSION="22.12"
 
 # email adress for notifications
 MAIL=jdoe@example.com
@@ -28,7 +26,10 @@ MAIL=jdoe@example.com
 # list of hosts (hostname or IP) and SSH port for Rsync
 SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX"
 
-# Should we fallback on servers when the first is unreachable ?
+# explicit PATH
+PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin
+
+# Should we fallback on other servers when the first one is unreachable?
 SERVERS_FALLBACK=${SERVERS_FALLBACK:-1}
 
 # timeout (in seconds) for SSH connections
@@ -47,31 +48,479 @@ PIDFILE="/var/run/${PROGNAME}.pid"
 # Customize the log path if you have multiple scripts and with separate logs
 LOGFILE="/var/log/evobackup.log"
 
-# Enable/Disable tasks
-LOCAL_TASKS=${LOCAL_TASKS:-1}
-SYNC_TASKS=${SYNC_TASKS:-1}
+# Full Rsync log file, reset each time
+RSYNC_LOGFILE="/var/log/${PROGNAME}.rsync.log"
 
 HOSTNAME=$(hostname)
 
-##### SETUP AND FUNCTIONS #############################################
-
-START_EPOCH=$(/bin/date +%s)
 DATE_FORMAT="%Y-%m-%d %H:%M:%S"
 
-# shellcheck disable=SC2174
-mkdir -p -m 700 ${LOCAL_BACKUP_DIR}
+# Enable/disable local tasks (default: enabled)
+: "${LOCAL_TASKS:=1}"
+# Enable/disable sync tasks (default: enabled)
+: "${SYNC_TASKS:=1}"
 
-PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin
+CANARY_FILE="/zzz_evobackup_canary"
 
-## lang = C for english outputs
-export LANGUAGE=C
-export LANG=C
+# Source paths can be customized
+# Empty lines, and lines containing # or ; are ignored
+RSYNC_INCLUDES="
+/etc
+/root
+/var
+/home
+"
 
-## Force umask
-umask 077
+# Excluded paths can be customized
+# Empty lines, and lines beginning with # or ; are ignored
+RSYNC_EXCLUDES="
+/dev
+/proc
+/run
+/sys
+/tmp
+/usr/doc
+/usr/obj
+/usr/share/doc
+/usr/src
+/var/apt
+/var/cache
+/var/db/munin/*.tmp
+/var/lib/amavis/amavisd.sock
+/var/lib/amavis/tmp
+/var/lib/clamav/*.tmp
+/var/lib/elasticsearch
+/var/lib/metche
+/var/lib/mongodb
+/var/lib/munin/*tmp*
+/var/lib/mysql
+/var/lib/php/sessions
+/var/lib/php5
+/var/lib/postgres
+/var/lib/postgresql
+/var/lib/sympa
+/var/lock
+/var/run
+/var/spool/postfix
+/var/spool/smtpd
+/var/spool/squid
+/var/state
+/var/tmp
+lost+found
+.nfs.*
+lxc/*/rootfs/tmp
+lxc/*/rootfs/usr/doc
+lxc/*/rootfs/usr/obj
+lxc/*/rootfs/usr/share/doc
+lxc/*/rootfs/usr/src
+lxc/*/rootfs/var/apt
+lxc/*/rootfs/var/cache
+lxc/*/rootfs/var/lib/php5
+lxc/*/rootfs/var/lib/php/sessions
+lxc/*/rootfs/var/lock
+lxc/*/rootfs/var/run
+lxc/*/rootfs/var/state
+lxc/*/rootfs/var/tmp
+/home/mysqltmp
+"
 
-## Initialize variable to store SSH connection errors
-SERVERS_SSH_ERRORS=""
+
+##### FUNCTIONS #######################################################
+
+local_tasks() {
+    log "START LOCAL_TASKS"
+
+    # You can comment or uncomment sections below to customize the backup
+
+    ## OpenLDAP : example with slapcat
+    # slapcat -n 0 -l ${LOCAL_BACKUP_DIR}/config.ldap.bak
+    # slapcat -n 1 -l ${LOCAL_BACKUP_DIR}/data.ldap.bak
+    # slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak
+
+    ## MySQL
+
+    ## Purge previous dumps
+    # rm -f  ${LOCAL_BACKUP_DIR}/mysql.*.gz
+    # rm -rf ${LOCAL_BACKUP_DIR}/mysql
+    # rm -rf ${LOCAL_BACKUP_DIR}/mysqlhotcopy
+    # rm -rf /home/mysqldump
+    # find ${LOCAL_BACKUP_DIR}/ -type f -name '*.err' -delete
+
+    ## example with global and compressed mysqldump
+    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
+    #  --opt --all-databases --force --events --hex-blob 2> ${LOCAL_BACKUP_DIR}/mysql.bak.err | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz
+    # last_rc=$?
+    # if [ ${last_rc} -ne 0 ]; then
+    #     error "mysqldump (global compressed) returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/mysql.bak.err"
+    #     rc=101
+    # fi
+
+    ## example with compressed SQL dump (with data) for each databases
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
+    # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
+    # | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)"); do
+    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i 2> ${LOCAL_BACKUP_DIR}/${i}.err | gzip --best > ${LOCAL_BACKUP_DIR}/mysql/${i}.sql.gz
+    #     last_rc=$?
+    #     if [ ${last_rc} -ne 0 ]; then
+    #         error "mysqldump (${i} compressed) returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/${i}.err"
+    #         rc=102
+    #     fi
+    # done
+
+    ## Dump all grants (requires 'percona-toolkit' package)
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
+    # pt-show-grants --flush --no-header 2> ${LOCAL_BACKUP_DIR}/mysql/all_grants.err > ${LOCAL_BACKUP_DIR}/mysql/all_grants.sql
+    # last_rc=$?
+    # if [ ${last_rc} -ne 0 ]; then
+    #     error "pt-show-grants returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/mysql/all_grants.err"
+    #     rc=103
+    # fi
+
+    # Dump all variables
+    # mysql -A -e"SHOW GLOBAL VARIABLES;" 2> ${LOCAL_BACKUP_DIR}/MySQLCurrentSettings.err > ${LOCAL_BACKUP_DIR}/MySQLCurrentSettings.txt
+    # last_rc=$?
+    # if [ ${last_rc} -ne 0 ]; then
+    #     error "mysql (variables) returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/MySQLCurrentSettings.err"
+    #     rc=104
+    # fi
+
+    ## example with SQL dump (schema only, no data) for each databases
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
+    # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
+    # | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)"); do
+    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases $i 2> ${LOCAL_BACKUP_DIR}/${i}.schema.err > ${LOCAL_BACKUP_DIR}/mysql/${i}.schema.sql
+    #     last_rc=$?
+    #     if [ ${last_rc} -ne 0 ]; then
+    #         error "mysqldump (${i} schema) returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/${i}.schema.err"
+    #         rc=105
+    #     fi
+    # done
+
+    ## example with *one* uncompressed SQL dump for *one* database (MYBASE)
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/MYBASE
+    # chown -RL mysql ${LOCAL_BACKUP_DIR}/mysql/
+    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -Q \
+    # --opt --events --hex-blob --skip-comments -T ${LOCAL_BACKUP_DIR}/mysql/MYBASE MYBASE 2> ${LOCAL_BACKUP_DIR}/mysql/MYBASE.err
+    # last_rc=$?
+    # if [ ${last_rc} -ne 0 ]; then
+    #     error "mysqldump (MYBASE) returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/mysql/MYBASE.err"
+    #     rc=106
+    # fi
+
+    ## example with two dumps for each table (.sql/.txt) for all databases
+    # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
+    # | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)" ); do
+    #     mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump
+    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \
+    #  --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i 2> /home/mysqldump/$i.err"
+    #     last_rc=$?
+    #     if [ ${last_rc} -ne 0 ]; then
+    #         error "mysqldump (${i} files) returned an error ${last_rc}, check /home/mysqldump/$i.err"
+    #         rc=107
+    #     fi
+    # done
+
+    ## example with mysqlhotcopy
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysqlhotcopy/
+    # mysqlhotcopy MYBASE ${LOCAL_BACKUP_DIR}/mysqlhotcopy/ 2> ${LOCAL_BACKUP_DIR}/mysqlhotcopy/MYBASE.err
+    # last_rc=$?
+    # if [ ${last_rc} -ne 0 ]; then
+    #     error "mysqlhotcopy returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/mysqlhotcopy/MYBASE.err"
+    #     rc=108
+    # fi
+
+    ## example for multiples MySQL instances
+    # mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3)
+    # grep --extended-regexp "^port\s*=\s*\d*" /etc/mysql/my.cnf | while read instance; do
+    #     instance=$(echo "$instance"|awk '{ print $3 }')
+    #     if [ "$instance" != "3306" ]
+    #     then
+    #         mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd 2> ${LOCAL_BACKUP_DIR}/mysql.${instance}.err | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.${instance}.bak.gz
+    #         last_rc=$?
+    #         if [ ${last_rc} -ne 0 ]; then
+    #             error "mysqldump (instance ${instance}) returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/mysql.${instance}.err"
+    #             rc=107
+    #         fi
+    #     fi
+    # done
+
+    ## PostgreSQL
+
+    ## Purge previous dumps
+    # rm -rf ${LOCAL_BACKUP_DIR}/pg.*.gz
+    # rm -rf ${LOCAL_BACKUP_DIR}/pg-backup.tar
+    # rm -rf ${LOCAL_BACKUP_DIR}/postgresql/*
+
+    ## example with pg_dumpall (warning: you need space in ~postgres)
+    # su - postgres -c "pg_dumpall > ~/pg.dump.bak"
+    # mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/
+
+    ## another method with gzip directly piped
+    # (
+    #    cd /var/lib/postgresql;
+    #    sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
+    # )
+
+    ## example with all tables from MYBASE excepts TABLE1 and TABLE2
+    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE
+
+    ## example with only TABLE1 and TABLE2 from MYBASE
+    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE
+
+    ## example with compressed PostgreSQL dump for each databases
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/postgresql
+    # chown postgres:postgres ${LOCAL_BACKUP_DIR}/postgresql
+    # (
+    #    cd /var/lib/postgresql
+    #    dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*)
+    #    for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done
+    # )
+
+    ## MongoDB
+
+    ## don't forget to create use with read-only access
+    ## > use admin
+    ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } )
+    ## Purge previous dumps
+    # rm -rf ${LOCAL_BACKUP_DIR}/mongodump/
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/
+    # mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/
+    # if [ $? -ne 0 ]; then
+    #   echo "Error with mongodump!"
+    # fi
+
+    ## Redis
+
+    ## Purge previous dumps
+    # rm -rf ${LOCAL_BACKUP_DIR}/redis/
+    # rm -rf ${LOCAL_BACKUP_DIR}/redis-*
+    ## Copy dump.rdb file for each found instance
+    # for instance in $(find /var/lib/ -mindepth 1 -maxdepth 1 -type d -name 'redis*'); do
+    #     if [ -f "${instance}/dump.rdb" ]; then
+    #         name=$(basename $instance)
+    #         mkdir -p ${LOCAL_BACKUP_DIR}/${name}
+    #         cp -a "${instance}/dump.rdb" "${LOCAL_BACKUP_DIR}/${name}"
+    #     fi
+    # done
+
+    ## ElasticSearch
+
+    ## Take a snapshot as a backup.
+    ## Warning: You need to have a path.repo configured.
+    ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes
+    # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" >> "${LOGFILE}"
+    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" >> "${LOGFILE}"
+    ## Clustered version here
+    ## It basically the same thing except that you need to check that NFS is mounted
+    # if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2'
+    # then
+    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" >> "${LOGFILE}"
+    #     curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" >> "${LOGFILE}"
+    # else
+    #     echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
+    # fi
+    ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
+    # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
+    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
+    # done
+    # date=$(/bin/date +%F)
+    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" >> "${LOGFILE}"
+
+    ## RabbitMQ
+
+    ## export config
+    # rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> "${LOGFILE}"
+
+    ## MegaCli config
+
+    # megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null
+
+    ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls)
+    network_targets="8.8.8.8 www.evolix.fr travaux.evolix.net"
+    mtr_bin=$(command -v mtr)
+    if [ -n "${mtr_bin}" ]; then
+        for addr in ${network_targets}; do
+            ${mtr_bin} -r "${addr}" > "${LOCAL_BACKUP_DIR}/mtr-${addr}"
+        done
+    fi
+    traceroute_bin=$(command -v traceroute)
+    if [ -n "${traceroute_bin}" ]; then
+        for addr in ${network_targets}; do
+            ${traceroute_bin} -n "${addr}" > "${LOCAL_BACKUP_DIR}/traceroute-${addr}" 2>&1
+        done
+    fi
+
+    server_state_dir="${LOCAL_BACKUP_DIR}/server-state"
+
+    dump_server_state_bin=$(command -v dump-server-state)
+    if [ -z "${dump_server_state_bin}" ]; then
+        error "dump-server-state is missing"
+        rc=1
+    else
+        if [ "${SYSTEM}" = "linux" ]; then
+            ${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
+            last_rc=$?
+            if [ ${last_rc} -ne 0 ]; then
+                error "dump-server-state returned an error ${last_rc}, check ${server_state_dir}"
+                rc=1
+            fi
+        else
+            ${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
+            last_rc=$?
+            if [ ${last_rc} -ne 0 ]; then
+                error "dump-server-state returned an error ${last_rc}, check ${server_state_dir}"
+                rc=1
+            fi
+        fi
+    fi
+
+    ## Dump rights
+    # getfacl -R /var  > ${server_state_dir}/rights-var.txt
+    # getfacl -R /etc  > ${server_state_dir}/rights-etc.txt
+    # getfacl -R /usr  > ${server_state_dir}/rights-usr.txt
+    # getfacl -R /home > ${server_state_dir}/rights-home.txt
+
+    log "STOP LOCAL_TASKS"
+}
+build_rsync_main_cmd() {
+    ###################################################################
+    # /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ #
+    ###################################################################
+    # DO NOT USE COMMENTS in rsync lines                              #
+    # DO NOT ADD WHITESPACES AFTER \ in rsync lines                   #
+    # It breaks the command and destroys data                         #
+    # You should not modify this, unless you are really REALLY sure   #
+    ###################################################################
+
+    # Create a temp file for excludes and includes
+    includes_file="$(mktemp "${PROGNAME}.includes.XXXXXX")"
+    excludes_file="$(mktemp "${PROGNAME}.excludes.XXXXXX")"
+    # … and add them to the list of files to delete at exit
+    temp_files="${temp_files} ${includes_file} ${excludes_file}"
+
+    # Store includes/excludes in files
+    # without blank lines of comments (# or ;)
+    echo "${RSYNC_INCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${includes_file}"
+    echo "${RSYNC_EXCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${excludes_file}"
+
+    # Rsync command
+    cmd="$(command -v rsync)"
+
+    # Rsync main options
+    cmd="${cmd} --archive"
+    cmd="${cmd} --itemize-changes"
+    cmd="${cmd} --quiet"
+    cmd="${cmd} --stats"
+    cmd="${cmd} --human-readable"
+    cmd="${cmd} --relative"
+    cmd="${cmd} --partial"
+    cmd="${cmd} --delete"
+    cmd="${cmd} --delete-excluded"
+    cmd="${cmd} --force"
+    cmd="${cmd} --ignore-errors"
+    cmd="${cmd} --log-file=${RSYNC_LOGFILE}"
+    cmd="${cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'"
+
+    # Rsync excludes
+    while read line ; do
+        cmd="${cmd} --exclude ${line}"
+    done < "${excludes_file}"
+
+    # Rsync local sources
+    cmd="${cmd} ${default_includes}"
+    while read line ; do
+        cmd="${cmd} ${line}"
+    done < "${includes_file}"
+
+    # Rsync remote destination
+    cmd="${cmd} root@${SSH_SERVER}:/var/backup/"
+
+    # output final command
+    echo "${cmd}"
+}
+build_rsync_canary_cmd() {
+    # Rsync command
+    cmd="$(command -v rsync)"
+    # Rsync options
+    cmd="${cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'"
+    # Rsync local source
+    cmd="${cmd} ${CANARY_FILE}"
+    # Rsync remote destination
+    cmd="${cmd} root@${SSH_SERVER}:/var/backup/"
+
+    # output final command
+    echo "${cmd}"
+}
+sync_tasks() {
+    n=0
+    server=""
+    if [ "${SERVERS_FALLBACK}" = "1" ]; then
+        # We try to find a suitable server
+        while :; do
+            server=$(pick_server "${n}")
+            test $? = 0 || exit 2
+
+            if test_server "${server}"; then
+                break
+            else
+                server=""
+                n=$(( n + 1 ))
+            fi
+        done
+    else
+        # we force the server
+        server=$(pick_server "${n}")
+    fi
+
+    SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
+    SSH_PORT=$(echo "${server}" | cut -d':' -f2)
+
+    log "START SYNC_TASKS - server=${server}"
+
+    # default paths, depending on system
+    if [ "${SYSTEM}" = "linux" ]; then
+        default_includes="/bin /boot /lib /opt /sbin /usr"
+    else
+        default_includes="/bsd /bin /sbin /usr"
+    fi
+
+    # reset Rsync log file
+    if [ -n "$(command -v truncate)" ]; then 
+        truncate -s 0 "${RSYNC_LOGFILE}"
+    else
+        printf "" > "${RSYNC_LOGFILE}"
+    fi
+
+    # Build the final Rsync command
+    rsync_main_cmd=$(build_rsync_main_cmd)
+
+    # … log it
+    log "SYNC_TASKS - Rsync main command : ${rsync_main_cmd}"
+
+    # … execute it
+    eval "${rsync_main_cmd}"
+
+    rsync_main_rc=$?
+
+    # Copy last lines of rsync log to the main log
+    tail -n 30 "${RSYNC_LOGFILE}" >> "${LOGFILE}"
+
+    if [ ${rsync_main_rc} -ne 0 ]; then
+        error "rsync returned an error ${rsync_main_rc}, check ${LOGFILE}"
+        rc=201
+    else
+        # Build the canary Rsync command
+        rsync_canary_cmd=$(build_rsync_canary_cmd)
+
+        # … log it
+        log "SYNC_TASKS - Rsync canary command : ${rsync_canary_cmd}"
+
+        # … execute it
+        eval "${rsync_canary_cmd}"
+    fi
+
+    log "STOP SYNC_TASKS - server=${server}"
+}
 
 # Call test_server with "HOST:PORT" string
 # It will return with 0 if the server is reachable.
@@ -133,391 +582,89 @@ log() {
         "$(/bin/date +"${DATE_FORMAT}")" "${PROGNAME}" "${pid}" "${msg}" \
         >> "${LOGFILE}"
 }
+error() {
+    msg="${1:-$(cat /dev/stdin)}"
+    pid=$$
+    printf "[%s] %s[%s]: %s\\n" \
+        "$(/bin/date +"${DATE_FORMAT}")" "${PROGNAME}" "${pid}" "${msg}" \
+        >&2
+}
 
-log "START GLOBAL - VERSION=${VERSION} LOCAL_TASKS=${LOCAL_TASKS} SYNC_TASKS=${SYNC_TASKS}"
+main() {
+    START_EPOCH=$(/bin/date +%s)
+    log "START GLOBAL - VERSION=${VERSION} LOCAL_TASKS=${LOCAL_TASKS} SYNC_TASKS=${SYNC_TASKS}"
 
-## Verify other evobackup process and kill if needed
-if [ -e "${PIDFILE}" ]; then
-    pid=$(cat "${PIDFILE}")
-    # Does process still exist ?
-    if kill -0 "${pid}" 2> /dev/null; then
-        # Killing the childs of evobackup.
-        for ppid in $(pgrep -P "${pid}"); do
-            kill -9 "${ppid}";
-        done
-        # Then kill the main PID.
-        kill -9 "${pid}"
-        printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\\n" >&2
-    else
-        rm -f "${PIDFILE}"
-    fi
-fi
-echo "$$" > "${PIDFILE}"
-# shellcheck disable=SC2064
-trap "rm -f ${PIDFILE}" EXIT
+    # shellcheck disable=SC2174
+    mkdir -p -m 700 ${LOCAL_BACKUP_DIR}
 
-##### LOCAL BACKUP ####################################################
+    ## Force umask
+    umask 077
 
-if [ "${LOCAL_TASKS}" = "1" ]; then
-    log "START LOCAL_TASKS"
+    ## Initialize variable to store SSH connection errors
+    SERVERS_SSH_ERRORS=""
 
-    # You can comment or uncomment sections below to customize the backup
-
-    ## OpenLDAP : example with slapcat
-    # slapcat -n 0 -l ${LOCAL_BACKUP_DIR}/config.ldap.bak
-    # slapcat -n 1 -l ${LOCAL_BACKUP_DIR}/data.ldap.bak
-    # slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak
-
-    ## MySQL
-
-    ## Purge previous dumps
-    # rm -f  ${LOCAL_BACKUP_DIR}/mysql.*.gz
-    # rm -rf ${LOCAL_BACKUP_DIR}/mysql
-    # rm -rf ${LOCAL_BACKUP_DIR}/mysqlhotcopy
-    # rm -rf /home/mysqldump
-
-    ## example with global and compressed mysqldump
-    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
-    #  --opt --all-databases --force --events --hex-blob | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz
-
-    ## example with compressed SQL dump (with data) for each databases
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
-    # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
-    # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
-    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i | gzip --best > ${LOCAL_BACKUP_DIR}/mysql/${i}.sql.gz
-    # done
-
-    ## Dump all grants (requires 'percona-toolkit' package)
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
-    # pt-show-grants --flush --no-header > ${LOCAL_BACKUP_DIR}/mysql/all_grants.sql
-
-    # Dump all variables
-    # mysql -A -e"SHOW GLOBAL VARIABLES;" > ${LOCAL_BACKUP_DIR}/MySQLCurrentSettings.txt
-
-    ## example with SQL dump (schema only, no data) for each databases
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
-    # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
-    # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
-    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases $i > ${LOCAL_BACKUP_DIR}/mysql/${i}.schema.sql
-    # done
-
-    ## example with *one* uncompressed SQL dump for *one* database (MYBASE)
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/MYBASE
-    # chown -RL mysql ${LOCAL_BACKUP_DIR}/mysql/
-    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -Q \
-    # --opt --events --hex-blob --skip-comments -T ${LOCAL_BACKUP_DIR}/mysql/MYBASE MYBASE
-
-    ## example with two dumps for each table (.sql/.txt) for all databases
-    # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
-    # | egrep -v "^(Database|information_schema|performance_schema|sys)" ); \
-    # do mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump ; \
-    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \
-    #  --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i; done
-
-    ## example with mysqlhotcopy
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysqlhotcopy/
-    # mysqlhotcopy MYBASE ${LOCAL_BACKUP_DIR}/mysqlhotcopy/
-
-    ## example for multiples MySQL instances
-    # mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3)
-    # grep -E "^port\s*=\s*\d*" /etc/mysql/my.cnf |while read instance; do
-    #     instance=$(echo "$instance"|awk '{ print $3 }')
-    #     if [ "$instance" != "3306" ]
-    #     then
-    #         mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.$instance.bak.gz
-    #     fi
-    # done
-
-    ## PostgreSQL
-
-    ## Purge previous dumps
-    # rm -rf ${LOCAL_BACKUP_DIR}/pg.*.gz
-    # rm -rf ${LOCAL_BACKUP_DIR}/pg-backup.tar
-    # rm -rf ${LOCAL_BACKUP_DIR}/postgresql/*
-    ## example with pg_dumpall (warning: you need space in ~postgres)
-    # su - postgres -c "pg_dumpall > ~/pg.dump.bak"
-    # mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/
-    ## another method with gzip directly piped
-    # cd /var/lib/postgresql
-    # sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
-    # cd - > /dev/null
-
-    ## example with all tables from MYBASE excepts TABLE1 and TABLE2
-    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE
-
-    ## example with only TABLE1 and TABLE2 from MYBASE
-    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE
-
-    ## example with compressed PostgreSQL dump for each databases
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/postgresql
-    # chown postgres:postgres ${LOCAL_BACKUP_DIR}/postgresql
-    # dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*)
-    # 
-    # for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done
-
-    ## MongoDB
-
-    ## don't forget to create use with read-only access
-    ## > use admin
-    ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } )
-    ## Purge previous dumps
-    # rm -rf ${LOCAL_BACKUP_DIR}/mongodump/
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/
-    # mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/
-    # if [ $? -ne 0 ]; then
-    #   echo "Error with mongodump!"
-    # fi
-
-    ## Redis
-
-    ## Purge previous dumps
-    # rm -rf ${LOCAL_BACKUP_DIR}/redis/
-    # rm -rf ${LOCAL_BACKUP_DIR}/redis-*
-    ## example with copy .rdb file
-    ## for the default instance :
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/redis/
-    # cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/redis/
-    ## for multiple instances :
-    # for instance in $(ls -d /var/lib/redis-*); do
-    #     name=$(basename $instance)
-    #     mkdir -p ${LOCAL_BACKUP_DIR}/${name}
-    #     cp -a ${instance}/dump.rdb ${LOCAL_BACKUP_DIR}/${name}
-    # done
-
-    ## ElasticSearch
-
-    ## Take a snapshot as a backup.
-    ## Warning: You need to have a path.repo configured.
-    ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes
-    # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log
-    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log
-    ## Clustered version here
-    ## It basically the same thing except that you need to check that NFS is mounted
-    # if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2'
-    # then
-    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log
-    #     curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log
-    # else
-    #     echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
-    # fi
-    ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
-    # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
-    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
-    # done
-    # date=$(/bin/date +%F)
-    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log
-
-    ## RabbitMQ
-
-    ## export config
-    #rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> $LOGFILE
-
-    ## MegaCli config
-
-    #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null
-
-    ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls)
-    for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do
-        mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr}
-        traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1
-    done
-
-    server_state_dir="${LOCAL_BACKUP_DIR}/server-state"
-
-    dump_server_state_bin=$(command -v dump-server-state)
-
-    if [ "${SYSTEM}" = "linux" ]; then
-        if [ -n "${dump_server_state_bin}" ]; then
-            ${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
-        else
-            mkdir -p "${server_state_dir}"
-
-            ## Dump system and kernel versions
-            uname -a > ${server_state_dir}/uname.txt
-
-            ## Dump process with ps
-            ps auwwx > ${server_state_dir}/ps.txt
-
-            ## Dump network connections with ss
-            ss -taupen > ${server_state_dir}/netstat.txt
-
-            ## List Debian packages
-            dpkg -l > ${server_state_dir}/packages
-            dpkg --get-selections > ${server_state_dir}/packages.getselections
-            apt-cache dumpavail > ${server_state_dir}/packages.available
-
-            ## Dump iptables
-            if [ -x /sbin/iptables ]; then
-                { /sbin/iptables -L -n -v; /sbin/iptables -t filter -L -n -v; } > ${server_state_dir}/iptables.txt
-            fi
-
-            ## Dump findmnt(8) output
-            FINDMNT_BIN=$(command -v findmnt)
-            if [ -x "${FINDMNT_BIN}" ]; then
-                ${FINDMNT_BIN} > ${server_state_dir}/findmnt.txt
-            fi
-
-            ## Dump MBR / table partitions
-            disks=$(lsblk -l | grep disk | grep -v -E '(drbd|fd[0-9]+)' | awk '{print $1}')
-            for disk in ${disks}; do
-                dd if="/dev/${disk}" of="${server_state_dir}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)"
-                fdisk -l "/dev/${disk}" > "${server_state_dir}/partitions-${disk}" 2>&1
+    ## Verify other evobackup process and kill if needed
+    if [ -e "${PIDFILE}" ]; then
+        pid=$(cat "${PIDFILE}")
+        # Does process still exist ?
+        if kill -0 "${pid}" 2> /dev/null; then
+            # Killing the childs of evobackup.
+            for ppid in $(pgrep -P "${pid}"); do
+                kill -9 "${ppid}";
             done
-            cat ${server_state_dir}/partitions-* > ${server_state_dir}/partitions
-        fi
-    else
-        if [ -n "${dump_server_state_bin}" ]; then
-            ${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
+            # Then kill the main PID.
+            kill -9 "${pid}"
+            printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\\n" >&2
         else
-            mkdir -p "${server_state_dir}"
-
-            ## Dump system and kernel versions
-            uname -a > ${server_state_dir}/uname
-
-            ## Dump process with ps
-            ps auwwx > ${server_state_dir}/ps.out
-
-            ## Dump network connections with fstat
-            fstat | head -1 > ${server_state_dir}/netstat.out
-            fstat | grep internet >> ${server_state_dir}/netstat.out
-
-            ## List OpenBSD packages
-            pkg_info -m > ${server_state_dir}/packages
-
-            ## Dump MBR / table partitions
-            disklabel sd0 > ${server_state_dir}/partitions
-
-            ## Dump pf infos
-            pfctl -sa > ${server_state_dir}/pfctl-sa.txt
+            rm -f "${PIDFILE}"
         fi
     fi
+    echo "$$" > "${PIDFILE}"
 
-    ## Dump rights
-    #getfacl -R /var  > ${server_state_dir}/rights-var.txt
-    #getfacl -R /etc  > ${server_state_dir}/rights-etc.txt
-    #getfacl -R /usr  > ${server_state_dir}/rights-usr.txt
-    #getfacl -R /home > ${server_state_dir}/rights-home.txt
+    # Initialize a list of files to delete at exit
+    # Any file added to the list will also be deleted at exit
+    temp_files="${PIDFILE}"
 
-    log "STOP LOCAL_TASKS"
-fi
-
-##### REMOTE BACKUP ###################################################
-
-
-if [ "${SYNC_TASKS}" = "1" ]; then
-    n=0
-    server=""
-    if [ "${SERVERS_FALLBACK}" = "1" ]; then
-        # We try to find a suitable server
-        while :; do
-            server=$(pick_server "${n}")
-            test $? = 0 || exit 2
-
-            if test_server "${server}"; then
-                break
-            else
-                server=""
-                n=$(( n + 1 ))
-            fi
-        done
-    else
-        # we force the server
-        server=$(pick_server "${n}")
-    fi
-
-    SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
-    SSH_PORT=$(echo "${server}" | cut -d':' -f2)
-
-    if [ "${SYSTEM}" = "linux" ]; then
-        rep="/bin /boot /lib /opt /sbin /usr"
-    else
-        rep="/bsd /bin /sbin /usr"
-    fi
-
-    log "START SYNC_TASKS - server=${server}"
+    # shellcheck disable=SC2064
+    trap "rm -f ${temp_files}" EXIT
 
+    # Update canary to keep track of each run
     update-evobackup-canary --who "${PROGNAME}"
 
-    # /!\ DO NOT USE COMMENTS in the rsync command /!\
-    # It breaks the command and destroys data, simply remove (or add) lines.
+    if [ "${LOCAL_TASKS}" = "1" ]; then
+        local_tasks
+    fi
 
-    # Remote shell command
-    RSH_COMMAND="ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'"
+    if [ "${SYNC_TASKS}" = "1" ]; then
+        sync_tasks
+    fi
 
-    # ignore check because we want it to split the different arguments to $rep
-    # shellcheck disable=SC2086
-    rsync -avzh --relative --stats --delete --delete-excluded --force --ignore-errors --partial \
-        --exclude "dev"                                    \
-        --exclude "lost+found"                             \
-        --exclude ".nfs.*"                                 \
-        --exclude "/usr/doc"                               \
-        --exclude "/usr/obj"                               \
-        --exclude "/usr/share/doc"                         \
-        --exclude "/usr/src"                               \
-        --exclude "/var/apt"                               \
-        --exclude "/var/cache"                             \
-        --exclude "/var/lib/amavis/amavisd.sock"           \
-        --exclude "/var/lib/amavis/tmp"                    \
-        --exclude "/var/lib/clamav/*.tmp"                  \
-        --exclude "/var/lib/elasticsearch"                 \
-        --exclude "/var/lib/metche"                        \
-        --exclude "/var/lib/munin/*tmp*"                   \
-        --exclude "/var/db/munin/*.tmp"                    \
-        --exclude "/var/lib/mysql"                         \
-        --exclude "/var/lib/php5"                          \
-        --exclude "/var/lib/php/sessions"                  \
-        --exclude "/var/lib/postgres"                      \
-        --exclude "/var/lib/postgresql"                    \
-        --exclude "/var/lib/sympa"                         \
-        --exclude "/var/lock"                              \
-        --exclude "/var/run"                               \
-        --exclude "/var/spool/postfix"                     \
-        --exclude "/var/spool/smtpd"                       \
-        --exclude "/var/spool/squid"                       \
-        --exclude "/var/state"                             \
-        --exclude "/var/tmp"                               \
-        --exclude "lxc/*/rootfs/tmp"                       \
-        --exclude "lxc/*/rootfs/usr/doc"                   \
-        --exclude "lxc/*/rootfs/usr/obj"                   \
-        --exclude "lxc/*/rootfs/usr/share/doc"             \
-        --exclude "lxc/*/rootfs/usr/src"                   \
-        --exclude "lxc/*/rootfs/var/apt"                   \
-        --exclude "lxc/*/rootfs/var/cache"                 \
-        --exclude "lxc/*/rootfs/var/lib/php5"              \
-        --exclude "lxc/*/rootfs/var/lib/php/sessions"      \
-        --exclude "lxc/*/rootfs/var/lock"                  \
-        --exclude "lxc/*/rootfs/var/log"                   \
-        --exclude "lxc/*/rootfs/var/run"                   \
-        --exclude "lxc/*/rootfs/var/state"                 \
-        --exclude "lxc/*/rootfs/var/tmp"                   \
-        --exclude "/home/mysqltmp"                         \
-            ${rep}                                         \
-            /etc                                           \
-            /root                                          \
-            /var                                           \
-            /home                                          \
-            /zzz_evobackup_canary                          \
-        -e "${RSH_COMMAND}"                                \
-        "root@${SSH_SERVER}:/var/backup/"                  \
-        | tail -30 >> $LOGFILE
+    STOP_EPOCH=$(/bin/date +%s)
 
-    log "STOP SYNC_TASKS - server=${server}"
-fi
+    if [ "${SYSTEM}" = "openbsd" ]; then
+        start_time=$(/bin/date -f "%s" -j "${START_EPOCH}" +"${DATE_FORMAT}")
+        stop_time=$(/bin/date -f "%s" -j "${STOP_EPOCH}" +"${DATE_FORMAT}")
+    else
+        start_time=$(/bin/date --date="@${START_EPOCH}" +"${DATE_FORMAT}")
+        stop_time=$(/bin/date --date="@${STOP_EPOCH}" +"${DATE_FORMAT}")
+    fi
+    duration=$(( STOP_EPOCH - START_EPOCH ))
 
-##### REPORTING #######################################################
+    log "STOP GLOBAL - start='${start_time}' stop='${stop_time}' duration=${duration}s"
 
-STOP_EPOCH=$(/bin/date +%s)
+    tail -20 "${LOGFILE}" | mail -s "[info] EvoBackup - Client ${HOSTNAME}" ${MAIL}
+}
 
-if [ "${SYSTEM}" = "openbsd" ]; then
-    start_time=$(/bin/date -f "%s" -j "${START_EPOCH}" +"${DATE_FORMAT}")
-    stop_time=$(/bin/date -f "%s" -j "${STOP_EPOCH}" +"${DATE_FORMAT}")
-else
-    start_time=$(/bin/date --date="@${START_EPOCH}" +"${DATE_FORMAT}")
-    stop_time=$(/bin/date --date="@${STOP_EPOCH}" +"${DATE_FORMAT}")
-fi
-duration=$(( STOP_EPOCH - START_EPOCH ))
+# set all programs to C language (english)
+export LC_ALL=C
 
-log "STOP GLOBAL - start='${start_time}' stop='${stop_time}' duration=${duration}s"
+# Error on unassigned variable
+set -u
 
-tail -20 "${LOGFILE}" \
-    | mail -s "[info] EvoBackup - Client ${HOSTNAME}" ${MAIL}
+# Default return-code (0 == succes)
+rc=0
+
+# execute main funciton
+main
+
+exit ${rc}

roles/evobackup/tasks/main.yml

@@ -1,4 +1,6 @@
 ---
+# Info : old zzz_evobackup used sh, and since "sh" match "bash", we keep "sh" for regex commands
+
 - name: "Copy zzz_evobackup script"
   copy:
     src: zzz_evobackup
@@ -10,6 +12,17 @@
   tags:
     - evobackup
 
+- name: "Copy update-evobackup-canary script"
+  copy:
+    src: 000-update-evobackup-canary
+    dest: /usr/share/scripts/000-update-evobackup-canary
+    owner: root
+    group: wheel
+    mode: "0755"
+    force: true
+  tags:
+    - evobackup
+
 - name: "Fetch daily.local content"
   command: 'grep "sh /usr/share/scripts/zzz_evobackup" /etc/daily.local'
   check_mode: false
@@ -22,13 +35,12 @@
 - name: "Add evobackup cron (disabled)"
   lineinfile:
     path: /etc/daily.local
-    line: '#sh /usr/share/scripts/zzz_evobackup'
+    line: '#bash /usr/share/scripts/zzz_evobackup'
     owner: root
     mode: "0600"
     create: true
   when:
-    - not (daily_local_content.stdout
-      | regex_search('sh /usr/share/scripts/zzz_evobackup'))
+    - not (daily_local_content.stdout | regex_search('sh /usr/share/scripts/zzz_evobackup'))
   tags:
     - evobackup
 
@@ -43,6 +55,17 @@
   tags:
     - evobackup
 
+- name: "Add canary cron"
+  lineinfile:
+    path: /etc/daily.local
+    line: 'sh /usr/share/scripts/000-update-evobackup-canary'
+    insertafter: 'next_part "EvoBackup output:"'
+    owner: root
+    mode: "0600"
+    create: true
+  tags:
+    - evobackup
+
 - name: "Delete legacy evobackup root crontab"
   lineinfile:
     path: /var/cron/tabs/root

roles/evocheck/files/evocheck.sh

@@ -3,7 +3,7 @@
 # EvoCheck
 # Script to verify compliance of an OpenBSD server powered by Evolix
 
-readonly VERSION="23.02"
+readonly VERSION="23.06"
 
 # base functions
 
@@ -51,6 +51,17 @@ is_installed(){
 
 # logging
 
+log() {
+    date=$(/bin/date +"${DATE_FORMAT}")
+    if [ "${1}" != '' ]; then
+        printf "[%s] %s: %s\\n" "$date" "${PROGNAME}" "${1}" >> "${LOGFILE}"
+    else
+        while read line; do
+            printf "[%s] %s: %s\\n" "$date" "${PROGNAME}" "${line}" >> "${LOGFILE}"
+        done < /dev/stdin
+    fi
+}
+
 failed() {
     check_name=$1
     shift
@@ -64,6 +75,9 @@ failed() {
             printf "%s FAILED!\n" "${check_name}" 2>&1
         fi
     fi
+
+    # Always log verbose
+    log "${check_name} FAILED! ${check_comments}"
 }
 
 # check functions
@@ -104,9 +118,9 @@ check_raidok(){
 }
 check_evobackup(){
     if [ -f /etc/daily.local ]; then
-        grep -qE "^sh /usr/share/scripts/zzz_evobackup" /etc/daily.local || failed "IS_EVOBACKUP" "Make sure 'sh /usr/share/scripts/zzz_evobackup' is present and activated in /etc/daily.local"
+        grep -qE "^(ba)?sh /usr/share/scripts/zzz_evobackup" /etc/daily.local || failed "IS_EVOBACKUP" "Make sure 'bash or sh /usr/share/scripts/zzz_evobackup' is present and activated in /etc/daily.local"
     else
-        failed "IS_EVOBACKUP" "Make sure /etc/daily.local exists and 'sh /usr/share/scripts/zzz_evobackup' is present and activated in /etc/daily.local"
+        failed "IS_EVOBACKUP" "Make sure /etc/daily.local exists and 'bash or sh /usr/share/scripts/zzz_evobackup' is present and activated in /etc/daily.local"
     fi
 }
 check_uptodate(){
@@ -154,9 +168,9 @@ check_carpadvbase(){
     if ls /etc/hostname.carp* 1> /dev/null 2>&1; then
         bad_advbase=0
         for advbase in $(ifconfig carp | grep advbase | awk -F 'advbase' '{print $2}' | awk '{print $1}' | xargs); do
-        if [ "$advbase" -gt 5 ]; then
-            bad_advbase=1
-        fi
+            if [ "$advbase" -gt 5 ]; then
+                bad_advbase=1
+            fi
         done
         if [ "$bad_advbase" -eq 1 ]; then
             failed "IS_CARPADVBASE" "At least one CARP interface has advbase greater than 5 seconds!"
@@ -172,7 +186,7 @@ check_carppreempt(){
         if [ -f /etc/sysctl.conf ]; then
             grep -qE "^net.inet.carp.preempt=1" /etc/sysctl.conf || failed "IS_CARPPREEMPT" "The preempt parameter is not permanently activated! Please add 'net.inet.carp.preempt=1' in /etc/sysctl.conf"
         else
-        failed "IS_CARPPREEMPT" "Make sure /etc/sysctl.conf exists and contains the line 'net.inet.carp.preempt=1'"
+            failed "IS_CARPPREEMPT" "Make sure /etc/sysctl.conf exists and contains the line 'net.inet.carp.preempt=1'"
         fi
     fi
 }
@@ -415,7 +429,7 @@ get_command() {
         evocheck) echo "${0}" ;;
         evomaintenance) command -v "evomaintenance.sh" ;;
         motd-carp-state) command -v "motd-carp-state.sh" ;;
-        
+
         ## General case, where the program name is the same as the command name
         *) command -v "${program}" ;;
     esac
@@ -512,6 +526,17 @@ check_root_user() {
         failed "IS_ROOT_USER" "root user should not have a password ; replace the password field with 'vipw' for the root user with '*************' (exactly 13 asterisks) "
     fi
 }
+check_mount(){
+    for fstab_entry in $(grep ffs /etc/fstab | grep -v "^#" | awk '{print $2}'); do
+        mount | awk '{print $3}' | grep -q "^$fstab_entry$" || failed "IS_MOUNT" "Local OpenBSD partition(s) detected in /etc/fstab but not mounted"
+    done
+}
+check_mountfstab() {
+    for mount_point in $(mount | awk '{print $3}'); do
+        grep -q " $mount_point " /etc/fstab || failed "IS_MOUNT_FSTAB" "Partition(s) detected mounted but no presence in /etc/fstab"
+    done
+}
+
 
 main() {
     # Default return code : 0 = no error
@@ -561,24 +586,34 @@ main() {
     test "${IS_EVOLIX_USER:=1}" = 1 && check_evolix_user
     test "${IS_CHECK_VERSIONS:=1}" = 1 && check_versions
     test "${IS_ROOT_USER:=1}" = 1 && check_root_user
+    test "${IS_MOUNT:=1}" = 1 && check_mount
+    test "${IS_MOUNT_FSTAB:=1}" = 1 && check_mountfstab
 
     exit ${RC}
 }
-cleanup_temp_files() {
+cleanup() {
+    # Cleanup tmp files
     # shellcheck disable=SC2086
     rm -f ${files_to_cleanup}
+
+    log "$PROGNAME exit."
 }
 
+PROGNAME=$(basename "$0")
+
+LOGFILE="/var/log/evocheck.log"
+
+CONFIGFILE="/etc/evocheck.cf"
+
+DATE_FORMAT="%Y-%m-%d %H:%M:%S"
+
 # Disable LANG*
 export LANG=C
 export LANGUAGE=C
 
-files_to_cleanup=""
-trap cleanup_temp_files 0
-
 # Source configuration file
 # shellcheck disable=SC1091
-test -f /etc/evocheck.cf && . /etc/evocheck.cf
+test -f "${CONFIGFILE}" && . "${CONFIGFILE}"
 
 # Parse options
 # based on https://gist.github.com/deshion/10d3cb5f88a21671e17a
@@ -621,5 +656,25 @@ while :; do
     shift
 done
 
+# Keep this after "show_version(); exit 0" which is called by check_versions
+# to avoid logging exit twice.
+files_to_cleanup=""
+trap cleanup EXIT INT TERM
+
+log '-----------------------------------------------'
+log "Running $PROGNAME $VERSION..."
+
+# Log config file content
+if [ -f "${CONFIGFILE}" ]; then
+    log "Runtime configuration (${CONFIGFILE}):"
+    conf=$(sed -e '/^[[:blank:]]*#/d; s/#.*//; /^[[:blank:]]*$/d' "${CONFIGFILE}")
+    if [ ! -z "${conf}" ]; then
+        sed -e '/^[[:blank:]]*#/d; s/#.*//; /^[[:blank:]]*$/d' "${CONFIGFILE}" | log
+    else
+        log "${CONFIGFILE} is empty."
+    fi
+fi
+
 # shellcheck disable=SC2086
 main ${ARGS}
+

roles/nagios-nrpe/tasks/main.yml

@@ -46,13 +46,33 @@
   tags:
     - nagios-nrpe
 
-- name: "Allow NRPE hosts"
+- name: "Fetch nrpe config content"
+  command: 'grep "allowed_hosts=" /etc/nrpe.d/evolix.cfg'
+  check_mode: false
+  register: nrpe_config_content
+  failed_when: false
+  changed_when: false
+  tags:
+    - nagios-nrpe
+
+- name: "Allow NRPE hosts - if no allowed_hosts configured"
   lineinfile:
     dest: /etc/nrpe.d/evolix.cfg
     insertbefore: BOF
+    regex: "allowed_hosts={{ nagios_nrpe_allowed_hosts | join(',') }}"
+    line: 'allowed_hosts={{ nagios_nrpe_allowed_hosts | join(",") }}'
+  when: nrpe_config_content.rc != 0
+  tags:
+    - nagios-nrpe
+
+- name: "Allow NRPE hosts - if allowed_hosts already configured : keep added IP"
+  lineinfile:
+    dest: /etc/nrpe.d/evolix.cfg
     backrefs: true
-    regex: "allowed_hosts={{ nagios_nrpe_allowed_hosts | join(',') }}(.*)"
+    insertbefore: BOF
+    regex: "allowed_hosts={{ nagios_nrpe_allowed_hosts | join(',') }}(.*)*"
     line: 'allowed_hosts={{ nagios_nrpe_allowed_hosts | join(",") }}\1'
+  when: nrpe_config_content.rc == 0
   tags:
     - nagios-nrpe
 

roles/nagios-nrpe/templates/evolix_bsd.cfg.j2

@@ -1,3 +1,8 @@
+{% if ansible_distribution_version is version_compare("7.2",'>=') %}
+# Allow older cipher suites for older Icinga version
+ssl_cipher_list=ALL:!MD5:@STRENGTH:@SECLEVEL=0
+
+{% endif %}
 # System checks
 command[check_users]=/usr/local/libexec/nagios/check_users -w 5 -c 10
 command[check_load]=/usr/local/libexec/nagios/check_load --percpu --warning=0.7,0.6,0.5 --critical=0.9,0.8,0.7
@@ -30,7 +35,7 @@ command[check_bioctl]=/usr/local/libexec/nagios/check_bioctl -d sd2
 command[check_openvpn_certificates]=doas /usr/local/libexec/nagios/plugins/check_openvpn_certificates.sh
 #command[check_carp0]=/usr/local/libexec/nagios/plugins/check_carp_if carp0 master
 command[check_mem]=/usr/local/libexec/nagios/plugins/check_free_mem.sh -w 20% -c 10%
-#command[check_vpn]=doas /usr/local/libexec/nagios/plugins/check_ipsecctl.sh IPDISTANTE IPLOCALE "VPN MARSEILLE-ROME"
+#command[check_vpn]=/usr/local/libexec/nagios/plugins/check_ipsecctl_critiques.sh
 command[check_pf_states]=doas /usr/local/libexec/nagios/plugins/check_pf_states
 command[check_ospfd]=doas /usr/local/libexec/nagios/plugins/check_ospfd
 command[check_ospf6d]=doas /usr/local/libexec/nagios/plugins/check_ospf6d

roles/post-install/tasks/motd.yml

@@ -27,6 +27,7 @@
   cron:
     name: dynamic motd for CARP
     job: "/bin/sh /usr/share/scripts/motd-carp-state.sh"
+    minute: "0"
     disabled: true
   when:
     - not (root_crontab_content.stdout

roles/post-install/templates/generateldif.sh.j2

@@ -147,6 +147,13 @@ ServiceName: packetfilter
 ServiceType: firewall
 ServiceVersion: packetfilter
 
+dn: ServiceName=pf_states,EvoComputerName=${EvoComputerName},ou=computer,dc=evolix,dc=net
+NagiosEnabled: TRUE
+objectClass: EvoService
+ServiceName: pf_states
+ServiceType: firewall
+ServiceVersion: pf_states
+
 EOT
 
 if grep -Eq 'sd.*RAID' /var/run/dmesg.boot; then