Merge pull request 'fix-tempfiles' (#72) from fix-tempfiles into master

Reviewed-on: evolix/evobackup#72

.Jenkinsfile

@@ -0,0 +1,33 @@
+pipeline {
+    agent { label 'sbuild' }
+    stages {
+        stage('Build Debian package') {
+            when {
+                branch 'debian'
+            }
+            steps {
+                script {
+                    sh 'gbp buildpackage'
+                }
+                archiveArtifacts allowEmptyArchive: true, artifacts: 'build-area/*.gz,build-area/*.bz2,build-area/*.xz,build-area/*.deb,build-area/*.dsc,build-area/*.changes,build-area/*.buildinfo,build-area/*.build,build-area/lintian.txt'
+            }
+        }
+
+        stage('Upload Debian package') {
+            when {
+                branch 'debian'
+            }
+            steps {
+                script {
+                    sh 'rsync -avP build-area/bkctld*.deb build-area/bkctld*.changes build-area/bkctld*.buildinfo pub.evolix.org:/srv/upload/'
+                }
+            }
+        }
+    }
+    post {
+        // Clean after build
+        always {
+            cleanWs()
+        }
+    }
+}

.drone.yml

@@ -1,41 +0,0 @@
-kind: pipeline
-name: default
-
-steps:
-- name: build debian package
-  image: evolix/gbp:latest
-  branches:
-  - debian
-  commands:
-  - mk-build-deps --install --remove debian/control
-  - git clean --force
-  - gbp buildpackage
-  volumes:
-  - name: tmp
-    path: /tmp
-  when:
-    branch:
-    - debian
-
-- name: upload debian package
-  image: drillster/drone-rsync
-  settings:
-    hosts: ["pub.evolix.net"]
-    port: 22
-    user: droneci
-    key:
-      from_secret: drone_private_key
-    target: /home/droneci/bkctld/
-    source: /tmp/bkctld/
-    delete: true
-  volumes:
-  - name: tmp
-    path: /tmp
-  when:
-    branch:
-    - debian
-
-volumes:
-- name: tmp
-  host:
-    path: /tmp

.gitignore

@@ -0,0 +1,2 @@
+*.swp
+.vagrant

client/CHANGELOG.md

@@ -10,18 +10,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-* Use --dump-dir instead of --backup-dir to supress dump-server-state warning
-
 ### Deprecated
 
 ### Removed
 
 ### Fixed
 
-* Make start_time and stop_time compatible with OpenBSD
-
 ### Security
 
+## [22.12]
+### Changed
+
+* Use --dump-dir instead of --backup-dir to suppress dump-server-state warning
+* Do not use rsync compression
+* Replace rsync option --verbose by --itemize-changes
+* Add canary to zzz_evobackup
+* update-evobackup-canary: do not use GNU date, for it to be compatible with OpenBSD
+* Add AGPL License and README
+* Script now depends on Bash
+* tolerate absence of mtr or traceroute
+* Only one loop for all Redis instances
+* remodel how we build the rsync command
+* use sub shells instead of moving around
+* Separate Rsync for the canary file if the main Rsync has finished without errors
+
+### Removed
+
+* No more fallback if dump-server-state is missing
+
+### Fixed
+
+* Make start_time and stop_time compatible with OpenBSD
+
 ## [22.03]
 
 Split client and server parts of the project

client/README.md

@@ -0,0 +1,3 @@
+Pour l'installation de `zzz_evobackup`, voir <https://intra.evolix.net/OutilsInternes/EvoBackupClient#installer-et-configurer-le-client-evobackup>
+
+Pour `update-evobackup-canary`, voir <https://intra.evolix.net/OutilsInternes/update-evobackup-canary>

client/update-evobackup-canary

@@ -0,0 +1,129 @@
+#!/bin/sh
+
+PROGNAME="update-evobackup-canary"
+REPOSITORY="https://gitea.evolix.org/evolix/evobackup"
+
+VERSION="22.06"
+readonly VERSION
+
+# base functions
+
+show_version() {
+    cat <<END
+${PROGNAME} version ${VERSION}
+
+Copyright 2022 Evolix <info@evolix.fr>,
+               Jérémy Lecour <jlecour@evolix.fr>,
+               and others.
+
+${REPOSITORY} 
+
+${PROGNAME} comes with ABSOLUTELY NO WARRANTY. This is free software,
+and you are welcome to redistribute it under certain conditions.
+See the GNU General Public License v3.0 for details.
+END
+}
+show_help() {
+    cat <<END
+${PROGNAME} is updating a canary file for evobackup.
+
+Usage: ${PROGNAME} [OPTIONS]
+
+Main options
+ -w, --who      who has updated the file (default: logname())
+ -f, --file     path of the canary file (default: /zzz_evobackup_canary)
+ -V, --version  print version and exit
+ -h, --help     print this message and exit
+END
+}
+
+main() {
+    if [ -z "${who:-}" ]; then
+        who=$(logname)
+    fi
+    if [ -z "${canary_file:-}" ]; then
+        canary_file="/zzz_evobackup_canary"
+    fi
+    # This option is supported both on OpenBSD which does not use GNU date and on Debian
+    date=$(date "+%FT%T%z")
+
+    printf "%s %s\n" "${date}" "${who}" >> "${canary_file}"
+}
+
+# parse options
+# based on https://gist.github.com/deshion/10d3cb5f88a21671e17a
+while :; do
+    case $1 in
+        -h|-\?|--help)
+            show_help
+            exit 0
+            ;;
+        -V|--version)
+            show_version
+            exit 0
+            ;;
+
+        -w|--who)
+            # with value separated by space
+            if [ -n "$2" ]; then
+                who=$2
+                shift
+            else
+                printf 'ERROR: "-w|--who" requires a non-empty option argument.\n' >&2
+                exit 1
+            fi
+            ;;
+        --who=?*)
+            # with value speparated by =
+            who=${1#*=}
+            ;;
+        --who=)
+            # without value
+            printf 'ERROR: "--who" requires a non-empty option argument.\n' >&2
+            exit 1
+            ;;
+
+        -f|--file)
+            # with value separated by space
+            if [ -n "$2" ]; then
+                canary_file=$2
+                shift
+            else
+                printf 'ERROR: "-f|--file" requires a non-empty option argument.\n' >&2
+                exit 1
+            fi
+            ;;
+        --file=?*)
+            # with value speparated by =
+            canary_file=${1#*=}
+            ;;
+        --file=)
+            # without value
+            printf 'ERROR: "--file" requires a non-empty option argument.\n' >&2
+            exit 1
+            ;;
+
+        --)
+            # End of all options.
+            shift
+            break
+            ;;
+        -?*)
+            # ignore unknown options
+            printf 'WARN: Unknown option : %s\n' "$1" >&2
+            exit 1
+            ;;
+        *)
+            # Default case: If no more options then break out of the loop.
+            break
+            ;;
+    esac
+
+    shift
+done
+
+export LC_ALL=C
+
+set -u
+
+main

client/zzz_evobackup

@@ -1,26 +1,24 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Script Evobackup client
 # See https://gitea.evolix.org/evolix/evobackup
 #
-# Author: Gregory Colpart <reg@evolix.fr>
-# Contributors:
-# Romain Dessort <rdessort@evolix.fr>
-# Benoît Série <bserie@evolix.fr>
-# Tristan Pilat <tpilat@evolix.fr>
-# Victor Laborie <vlaborie@evolix.fr>
-# Jérémy Lecour <jlecour@evolix.fr>
+# Authors: Evolix <info@evolix.fr>,
+#          Gregory Colpart <reg@evolix.fr>,
+#          Romain Dessort <rdessort@evolix.fr>,
+#          Benoit Série <bserie@evolix.fr>,
+#          Tristan Pilat <tpilat@evolix.fr>,
+#          Victor Laborie <vlaborie@evolix.fr>,
+#          Jérémy Lecour <jlecour@evolix.fr>
+#          and others.
 #
 # Licence: AGPLv3
 #
 # /!\ DON'T FORGET TO SET "MAIL" and "SERVERS" VARIABLES
 
-# Fail on unassigned variables
-set -u
-
 ##### Configuration ###################################################
 
-VERSION="22.03"
+VERSION="22.12"
 
 # email adress for notifications
 MAIL=jdoe@example.com
@@ -28,7 +26,10 @@ MAIL=jdoe@example.com
 # list of hosts (hostname or IP) and SSH port for Rsync
 SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX"
 
-# Should we fallback on servers when the first is unreachable ?
+# explicit PATH
+PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin
+
+# Should we fallback on other servers when the first one is unreachable?
 SERVERS_FALLBACK=${SERVERS_FALLBACK:-1}
 
 # timeout (in seconds) for SSH connections
@@ -47,31 +48,486 @@ PIDFILE="/var/run/${PROGNAME}.pid"
 # Customize the log path if you have multiple scripts and with separate logs
 LOGFILE="/var/log/evobackup.log"
 
-# Enable/Disable tasks
-LOCAL_TASKS=${LOCAL_TASKS:-1}
-SYNC_TASKS=${SYNC_TASKS:-1}
+# Full Rsync log file, reset each time
+RSYNC_LOGFILE="/var/log/${PROGNAME}.rsync.log"
 
 HOSTNAME=$(hostname)
 
-##### SETUP AND FUNCTIONS #############################################
-
-START_EPOCH=$(/bin/date +%s)
 DATE_FORMAT="%Y-%m-%d %H:%M:%S"
 
-# shellcheck disable=SC2174
-mkdir -p -m 700 ${LOCAL_BACKUP_DIR}
+# Enable/disable local tasks (default: enabled)
+: "${LOCAL_TASKS:=1}"
+# Enable/disable sync tasks (default: enabled)
+: "${SYNC_TASKS:=1}"
 
-PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin
+CANARY_FILE="/zzz_evobackup_canary"
 
-## lang = C for english outputs
-export LANGUAGE=C
-export LANG=C
+# Source paths can be customized
+# Empty lines, and lines containing # or ; are ignored
+# NOTE: remember to single-quote paths if they contain globs (*)
+#       and you want to defer expansion
+RSYNC_INCLUDES="
+/etc
+/root
+/var
+/home
+"
 
-## Force umask
-umask 077
+# Excluded paths can be customized
+# Empty lines, and lines beginning with # or ; are ignored
+# NOTE: remember to single-quote paths if they contain globs (*)
+#       and you want to defer expansion
+RSYNC_EXCLUDES="
+/dev
+/proc
+/run
+/sys
+/tmp
+/usr/doc
+/usr/obj
+/usr/share/doc
+/usr/src
+/var/apt
+/var/cache
+'/var/db/munin/*.tmp'
+/var/lib/amavis/amavisd.sock
+/var/lib/amavis/tmp
+/var/lib/amavis/virusmails
+'/var/lib/clamav/*.tmp'
+/var/lib/elasticsearch
+/var/lib/metche
+/var/lib/mongodb
+'/var/lib/munin/*tmp*'
+/var/lib/mysql
+/var/lib/php/sessions
+/var/lib/php5
+/var/lib/postgres
+/var/lib/postgresql
+/var/lib/sympa
+/var/lock
+/var/run
+/var/spool/postfix
+/var/spool/smtpd
+/var/spool/squid
+/var/state
+/var/tmp
+lost+found
+'.nfs.*'
+'lxc/*/rootfs/tmp'
+'lxc/*/rootfs/usr/doc'
+'lxc/*/rootfs/usr/obj'
+'lxc/*/rootfs/usr/share/doc'
+'lxc/*/rootfs/usr/src'
+'lxc/*/rootfs/var/apt'
+'lxc/*/rootfs/var/cache'
+'lxc/*/rootfs/var/lib/php5'
+'lxc/*/rootfs/var/lib/php/sessions'
+'lxc/*/rootfs/var/lock'
+'lxc/*/rootfs/var/run'
+'lxc/*/rootfs/var/state'
+'lxc/*/rootfs/var/tmp'
+/home/mysqltmp
+"
 
-## Initialize variable to store SSH connection errors
-SERVERS_SSH_ERRORS=""
+
+##### FUNCTIONS #######################################################
+
+local_tasks() {
+    log "START LOCAL_TASKS"
+
+    # You can comment or uncomment sections below to customize the backup
+
+    ## OpenLDAP : example with slapcat
+    # slapcat -n 0 -l ${LOCAL_BACKUP_DIR}/config.ldap.bak
+    # slapcat -n 1 -l ${LOCAL_BACKUP_DIR}/data.ldap.bak
+    # slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak
+
+    ## MySQL
+
+    ## Purge previous dumps
+    # rm -f  ${LOCAL_BACKUP_DIR}/mysql.*.gz
+    # rm -rf ${LOCAL_BACKUP_DIR}/mysql
+    # rm -rf ${LOCAL_BACKUP_DIR}/mysqlhotcopy
+    # rm -rf /home/mysqldump
+    # find ${LOCAL_BACKUP_DIR}/ -type f -name '*.err' -delete
+
+    ## example with global and compressed mysqldump
+    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
+    #  --opt --all-databases --force --events --hex-blob 2> ${LOCAL_BACKUP_DIR}/mysql.bak.err | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz
+    # last_rc=$?
+    # if [ ${last_rc} -ne 0 ]; then
+    #     error "mysqldump (global compressed) returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/mysql.bak.err"
+    #     rc=101
+    # fi
+
+    ## example with compressed SQL dump (with data) for each databases
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
+    # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
+    # | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)"); do
+    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i 2> ${LOCAL_BACKUP_DIR}/${i}.err | gzip --best > ${LOCAL_BACKUP_DIR}/mysql/${i}.sql.gz
+    #     last_rc=$?
+    #     if [ ${last_rc} -ne 0 ]; then
+    #         error "mysqldump (${i} compressed) returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/${i}.err"
+    #         rc=102
+    #     fi
+    # done
+
+    ## Dump all grants (requires 'percona-toolkit' package)
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
+    # pt-show-grants --flush --no-header 2> ${LOCAL_BACKUP_DIR}/mysql/all_grants.err > ${LOCAL_BACKUP_DIR}/mysql/all_grants.sql
+    # last_rc=$?
+    # if [ ${last_rc} -ne 0 ]; then
+    #     error "pt-show-grants returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/mysql/all_grants.err"
+    #     rc=103
+    # fi
+
+    # Dump all variables
+    # mysql -A -e"SHOW GLOBAL VARIABLES;" 2> ${LOCAL_BACKUP_DIR}/MySQLCurrentSettings.err > ${LOCAL_BACKUP_DIR}/MySQLCurrentSettings.txt
+    # last_rc=$?
+    # if [ ${last_rc} -ne 0 ]; then
+    #     error "mysql (variables) returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/MySQLCurrentSettings.err"
+    #     rc=104
+    # fi
+
+    ## example with SQL dump (schema only, no data) for each databases
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
+    # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
+    # | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)"); do
+    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases $i 2> ${LOCAL_BACKUP_DIR}/${i}.schema.err > ${LOCAL_BACKUP_DIR}/mysql/${i}.schema.sql
+    #     last_rc=$?
+    #     if [ ${last_rc} -ne 0 ]; then
+    #         error "mysqldump (${i} schema) returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/${i}.schema.err"
+    #         rc=105
+    #     fi
+    # done
+
+    ## example with *one* uncompressed SQL dump for *one* database (MYBASE)
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/MYBASE
+    # chown -RL mysql ${LOCAL_BACKUP_DIR}/mysql/
+    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -Q \
+    # --opt --events --hex-blob --skip-comments -T ${LOCAL_BACKUP_DIR}/mysql/MYBASE MYBASE 2> ${LOCAL_BACKUP_DIR}/mysql/MYBASE.err
+    # last_rc=$?
+    # if [ ${last_rc} -ne 0 ]; then
+    #     error "mysqldump (MYBASE) returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/mysql/MYBASE.err"
+    #     rc=106
+    # fi
+
+    ## example with two dumps for each table (.sql/.txt) for all databases
+    # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
+    # | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)" ); do
+    #     mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump
+    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \
+    #  --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i 2> /home/mysqldump/$i.err"
+    #     last_rc=$?
+    #     if [ ${last_rc} -ne 0 ]; then
+    #         error "mysqldump (${i} files) returned an error ${last_rc}, check /home/mysqldump/$i.err"
+    #         rc=107
+    #     fi
+    # done
+
+    ## example with mysqlhotcopy
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysqlhotcopy/
+    # mysqlhotcopy MYBASE ${LOCAL_BACKUP_DIR}/mysqlhotcopy/ 2> ${LOCAL_BACKUP_DIR}/mysqlhotcopy/MYBASE.err
+    # last_rc=$?
+    # if [ ${last_rc} -ne 0 ]; then
+    #     error "mysqlhotcopy returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/mysqlhotcopy/MYBASE.err"
+    #     rc=108
+    # fi
+
+    ## example for multiples MySQL instances
+    # mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3)
+    # grep --extended-regexp "^port\s*=\s*\d*" /etc/mysql/my.cnf | while read instance; do
+    #     instance=$(echo "$instance"|awk '{ print $3 }')
+    #     if [ "$instance" != "3306" ]
+    #     then
+    #         mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd 2> ${LOCAL_BACKUP_DIR}/mysql.${instance}.err | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.${instance}.bak.gz
+    #         last_rc=$?
+    #         if [ ${last_rc} -ne 0 ]; then
+    #             error "mysqldump (instance ${instance}) returned an error ${last_rc}, check ${LOCAL_BACKUP_DIR}/mysql.${instance}.err"
+    #             rc=107
+    #         fi
+    #     fi
+    # done
+
+    ## PostgreSQL
+
+    ## Purge previous dumps
+    # rm -rf ${LOCAL_BACKUP_DIR}/pg.*.gz
+    # rm -rf ${LOCAL_BACKUP_DIR}/pg-backup.tar
+    # rm -rf ${LOCAL_BACKUP_DIR}/postgresql/*
+
+    ## example with pg_dumpall (warning: you need space in ~postgres)
+    # su - postgres -c "pg_dumpall > ~/pg.dump.bak"
+    # mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/
+
+    ## another method with gzip directly piped
+    # (
+    #    cd /var/lib/postgresql;
+    #    sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
+    # )
+
+    ## example with all tables from MYBASE excepts TABLE1 and TABLE2
+    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE
+
+    ## example with only TABLE1 and TABLE2 from MYBASE
+    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE
+
+    ## example with compressed PostgreSQL dump for each databases
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/postgresql
+    # chown postgres:postgres ${LOCAL_BACKUP_DIR}/postgresql
+    # (
+    #    cd /var/lib/postgresql
+    #    dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*)
+    #    for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done
+    # )
+
+    ## MongoDB
+
+    ## don't forget to create use with read-only access
+    ## > use admin
+    ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } )
+    ## Purge previous dumps
+    # rm -rf ${LOCAL_BACKUP_DIR}/mongodump/
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/
+    # mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/
+    # if [ $? -ne 0 ]; then
+    #   echo "Error with mongodump!"
+    # fi
+
+    ## Redis
+
+    ## Purge previous dumps
+    # rm -rf ${LOCAL_BACKUP_DIR}/redis/
+    # rm -rf ${LOCAL_BACKUP_DIR}/redis-*
+    ## Copy dump.rdb file for each found instance
+    # for instance in $(find /var/lib/ -mindepth 1 -maxdepth 1 '(' -type d -o -type l ')' -name 'redis*'); do
+    #     if [ -f "${instance}/dump.rdb" ]; then
+    #         name=$(basename $instance)
+    #         mkdir -p ${LOCAL_BACKUP_DIR}/${name}
+    #         cp -a "${instance}/dump.rdb" "${LOCAL_BACKUP_DIR}/${name}"
+    #         gzip "${LOCAL_BACKUP_DIR}/${name}/dump.rdb"
+    #     fi
+    # done
+
+    ## ElasticSearch
+
+    ## Take a snapshot as a backup.
+    ## Warning: You need to have a path.repo configured.
+    ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes
+    # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" >> "${LOGFILE}"
+    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" >> "${LOGFILE}"
+    ## Clustered version here
+    ## It basically the same thing except that you need to check that NFS is mounted
+    # if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2'
+    # then
+    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" >> "${LOGFILE}"
+    #     curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" >> "${LOGFILE}"
+    # else
+    #     echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
+    # fi
+    ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
+    # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
+    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
+    # done
+    # date=$(/bin/date +%F)
+    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" >> "${LOGFILE}"
+
+    ## RabbitMQ
+
+    ## export config
+    # rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> "${LOGFILE}"
+
+    ## MegaCli config
+
+    # megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null
+
+    ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls)
+    network_targets="8.8.8.8 www.evolix.fr travaux.evolix.net"
+    mtr_bin=$(command -v mtr)
+    if [ -n "${mtr_bin}" ]; then
+        for addr in ${network_targets}; do
+            ${mtr_bin} -r "${addr}" > "${LOCAL_BACKUP_DIR}/mtr-${addr}"
+        done
+    fi
+    traceroute_bin=$(command -v traceroute)
+    if [ -n "${traceroute_bin}" ]; then
+        for addr in ${network_targets}; do
+            ${traceroute_bin} -n "${addr}" > "${LOCAL_BACKUP_DIR}/traceroute-${addr}" 2>&1
+        done
+    fi
+
+    server_state_dir="${LOCAL_BACKUP_DIR}/server-state"
+
+    dump_server_state_bin=$(command -v dump-server-state)
+    if [ -z "${dump_server_state_bin}" ]; then
+        error "dump-server-state is missing"
+        rc=1
+    else
+        if [ "${SYSTEM}" = "linux" ]; then
+            ${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
+            last_rc=$?
+            if [ ${last_rc} -ne 0 ]; then
+                error "dump-server-state returned an error ${last_rc}, check ${server_state_dir}"
+                rc=1
+            fi
+        else
+            ${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
+            last_rc=$?
+            if [ ${last_rc} -ne 0 ]; then
+                error "dump-server-state returned an error ${last_rc}, check ${server_state_dir}"
+                rc=1
+            fi
+        fi
+    fi
+
+    ## Dump rights
+    # getfacl -R /var  > ${server_state_dir}/rights-var.txt
+    # getfacl -R /etc  > ${server_state_dir}/rights-etc.txt
+    # getfacl -R /usr  > ${server_state_dir}/rights-usr.txt
+    # getfacl -R /home > ${server_state_dir}/rights-home.txt
+
+    log "STOP LOCAL_TASKS"
+}
+build_rsync_main_cmd() {
+    ###################################################################
+    # /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ #
+    ###################################################################
+    # DO NOT USE COMMENTS in rsync lines                              #
+    # DO NOT ADD WHITESPACES AFTER \ in rsync lines                   #
+    # It breaks the command and destroys data                         #
+    # You should not modify this, unless you are really REALLY sure   #
+    ###################################################################
+
+    # Create a temp file for excludes and includes
+    includes_file="$(mktemp --tmpdir "${PROGNAME}.includes.XXXXXX")"
+    excludes_file="$(mktemp --tmpdir "${PROGNAME}.excludes.XXXXXX")"
+    # … and add them to the list of files to delete at exit
+    temp_files="${includes_file} ${excludes_file}"
+    trap "rm -f ${temp_files}" EXIT
+
+    # Store includes/excludes in files
+    # without blank lines of comments (# or ;)
+    echo "${RSYNC_INCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${includes_file}"
+    echo "${RSYNC_EXCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${excludes_file}"
+
+    # Rsync command
+    cmd="$(command -v rsync)"
+
+    # Rsync main options
+    cmd="${cmd} --archive"
+    cmd="${cmd} --itemize-changes"
+    cmd="${cmd} --quiet"
+    cmd="${cmd} --stats"
+    cmd="${cmd} --human-readable"
+    cmd="${cmd} --relative"
+    cmd="${cmd} --partial"
+    cmd="${cmd} --delete"
+    cmd="${cmd} --delete-excluded"
+    cmd="${cmd} --force"
+    cmd="${cmd} --ignore-errors"
+    cmd="${cmd} --log-file=${RSYNC_LOGFILE}"
+    cmd="${cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'"
+
+    # Rsync excludes
+    while read line ; do
+        cmd="${cmd} --exclude ${line}"
+    done < "${excludes_file}"
+
+    # Rsync local sources
+    cmd="${cmd} ${default_includes}"
+    while read line ; do
+        cmd="${cmd} ${line}"
+    done < "${includes_file}"
+
+    # Rsync remote destination
+    cmd="${cmd} root@${SSH_SERVER}:/var/backup/"
+
+    # output final command
+    echo "${cmd}"
+}
+build_rsync_canary_cmd() {
+    # Rsync command
+    cmd="$(command -v rsync)"
+    # Rsync options
+    cmd="${cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'"
+    # Rsync local source
+    cmd="${cmd} ${CANARY_FILE}"
+    # Rsync remote destination
+    cmd="${cmd} root@${SSH_SERVER}:/var/backup/"
+
+    # output final command
+    echo "${cmd}"
+}
+sync_tasks() {
+    n=0
+    server=""
+    if [ "${SERVERS_FALLBACK}" = "1" ]; then
+        # We try to find a suitable server
+        while :; do
+            server=$(pick_server "${n}")
+            test $? = 0 || exit 2
+
+            if test_server "${server}"; then
+                break
+            else
+                server=""
+                n=$(( n + 1 ))
+            fi
+        done
+    else
+        # we force the server
+        server=$(pick_server "${n}")
+    fi
+
+    SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
+    SSH_PORT=$(echo "${server}" | cut -d':' -f2)
+
+    log "START SYNC_TASKS - server=${server}"
+
+    # default paths, depending on system
+    if [ "${SYSTEM}" = "linux" ]; then
+        default_includes="/bin /boot /lib /opt /sbin /usr"
+    else
+        default_includes="/bsd /bin /sbin /usr"
+    fi
+
+    # reset Rsync log file
+    if [ -n "$(command -v truncate)" ]; then 
+        truncate -s 0 "${RSYNC_LOGFILE}"
+    else
+        printf "" > "${RSYNC_LOGFILE}"
+    fi
+
+    # Build the final Rsync command
+    rsync_main_cmd=$(build_rsync_main_cmd)
+
+    # … log it
+    log "SYNC_TASKS - Rsync main command : ${rsync_main_cmd}"
+
+    # … execute it
+    eval "${rsync_main_cmd}"
+
+    rsync_main_rc=$?
+
+    # Copy last lines of rsync log to the main log
+    tail -n 30 "${RSYNC_LOGFILE}" >> "${LOGFILE}"
+
+    if [ ${rsync_main_rc} -ne 0 ]; then
+        error "rsync returned an error ${rsync_main_rc}, check ${LOGFILE}"
+        rc=201
+    else
+        # Build the canary Rsync command
+        rsync_canary_cmd=$(build_rsync_canary_cmd)
+
+        # … log it
+        log "SYNC_TASKS - Rsync canary command : ${rsync_canary_cmd}"
+
+        # … execute it
+        eval "${rsync_canary_cmd}"
+    fi
+
+    log "STOP SYNC_TASKS - server=${server}"
+}
 
 # Call test_server with "HOST:PORT" string
 # It will return with 0 if the server is reachable.
@@ -133,388 +589,89 @@ log() {
         "$(/bin/date +"${DATE_FORMAT}")" "${PROGNAME}" "${pid}" "${msg}" \
         >> "${LOGFILE}"
 }
+error() {
+    msg="${1:-$(cat /dev/stdin)}"
+    pid=$$
+    printf "[%s] %s[%s]: %s\\n" \
+        "$(/bin/date +"${DATE_FORMAT}")" "${PROGNAME}" "${pid}" "${msg}" \
+        >&2
+}
 
-log "START GLOBAL - VERSION=${VERSION} LOCAL_TASKS=${LOCAL_TASKS} SYNC_TASKS=${SYNC_TASKS}"
+main() {
+    START_EPOCH=$(/bin/date +%s)
+    log "START GLOBAL - VERSION=${VERSION} LOCAL_TASKS=${LOCAL_TASKS} SYNC_TASKS=${SYNC_TASKS}"
 
-## Verify other evobackup process and kill if needed
-if [ -e "${PIDFILE}" ]; then
-    pid=$(cat "${PIDFILE}")
-    # Does process still exist ?
-    if kill -0 "${pid}" 2> /dev/null; then
-        # Killing the childs of evobackup.
-        for ppid in $(pgrep -P "${pid}"); do
-            kill -9 "${ppid}";
-        done
-        # Then kill the main PID.
-        kill -9 "${pid}"
-        printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\\n" >&2
-    else
-        rm -f "${PIDFILE}"
-    fi
-fi
-echo "$$" > "${PIDFILE}"
-# shellcheck disable=SC2064
-trap "rm -f ${PIDFILE}" EXIT
+    # shellcheck disable=SC2174
+    mkdir -p -m 700 ${LOCAL_BACKUP_DIR}
 
-##### LOCAL BACKUP ####################################################
+    ## Force umask
+    umask 077
 
-if [ "${LOCAL_TASKS}" = "1" ]; then
-    log "START LOCAL_TASKS"
+    ## Initialize variable to store SSH connection errors
+    SERVERS_SSH_ERRORS=""
 
-    # You can comment or uncomment sections below to customize the backup
-
-    ## OpenLDAP : example with slapcat
-    # slapcat -n 0 -l ${LOCAL_BACKUP_DIR}/config.ldap.bak
-    # slapcat -n 1 -l ${LOCAL_BACKUP_DIR}/data.ldap.bak
-    # slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak
-
-    ## MySQL
-
-    ## Purge previous dumps
-    # rm -f  ${LOCAL_BACKUP_DIR}/mysql.*.gz
-    # rm -rf ${LOCAL_BACKUP_DIR}/mysql
-    # rm -rf ${LOCAL_BACKUP_DIR}/mysqlhotcopy
-    # rm -rf /home/mysqldump
-
-    ## example with global and compressed mysqldump
-    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
-    #  --opt --all-databases --force --events --hex-blob | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz
-
-    ## example with compressed SQL dump (with data) for each databases
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
-    # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
-    # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
-    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i | gzip --best > ${LOCAL_BACKUP_DIR}/mysql/${i}.sql.gz
-    # done
-
-    ## Dump all grants (requires 'percona-toolkit' package)
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
-    # pt-show-grants --flush --no-header > ${LOCAL_BACKUP_DIR}/mysql/all_grants.sql
-
-    # Dump all variables
-    # mysql -A -e"SHOW GLOBAL VARIABLES;" > ${LOCAL_BACKUP_DIR}/MySQLCurrentSettings.txt
-
-    ## example with SQL dump (schema only, no data) for each databases
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
-    # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
-    # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
-    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases $i > ${LOCAL_BACKUP_DIR}/mysql/${i}.schema.sql
-    # done
-
-    ## example with *one* uncompressed SQL dump for *one* database (MYBASE)
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/MYBASE
-    # chown -RL mysql ${LOCAL_BACKUP_DIR}/mysql/
-    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -Q \
-    # --opt --events --hex-blob --skip-comments -T ${LOCAL_BACKUP_DIR}/mysql/MYBASE MYBASE
-
-    ## example with two dumps for each table (.sql/.txt) for all databases
-    # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
-    # | egrep -v "^(Database|information_schema|performance_schema|sys)" ); \
-    # do mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump ; \
-    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \
-    #  --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i; done
-
-    ## example with mysqlhotcopy
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysqlhotcopy/
-    # mysqlhotcopy MYBASE ${LOCAL_BACKUP_DIR}/mysqlhotcopy/
-
-    ## example for multiples MySQL instances
-    # mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3)
-    # grep -E "^port\s*=\s*\d*" /etc/mysql/my.cnf |while read instance; do
-    #     instance=$(echo "$instance"|awk '{ print $3 }')
-    #     if [ "$instance" != "3306" ]
-    #     then
-    #         mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.$instance.bak.gz
-    #     fi
-    # done
-
-    ## PostgreSQL
-
-    ## Purge previous dumps
-    # rm -rf ${LOCAL_BACKUP_DIR}/pg.*.gz
-    # rm -rf ${LOCAL_BACKUP_DIR}/pg-backup.tar
-    # rm -rf ${LOCAL_BACKUP_DIR}/postgresql/*
-    ## example with pg_dumpall (warning: you need space in ~postgres)
-    # su - postgres -c "pg_dumpall > ~/pg.dump.bak"
-    # mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/
-    ## another method with gzip directly piped
-    # cd /var/lib/postgresql
-    # sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
-    # cd - > /dev/null
-
-    ## example with all tables from MYBASE excepts TABLE1 and TABLE2
-    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE
-
-    ## example with only TABLE1 and TABLE2 from MYBASE
-    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE
-
-    ## example with compressed PostgreSQL dump for each databases
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/postgresql
-    # chown postgres:postgres ${LOCAL_BACKUP_DIR}/postgresql
-    # dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*)
-    # 
-    # for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done
-
-    ## MongoDB
-
-    ## don't forget to create use with read-only access
-    ## > use admin
-    ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } )
-    ## Purge previous dumps
-    # rm -rf ${LOCAL_BACKUP_DIR}/mongodump/
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/
-    # mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/
-    # if [ $? -ne 0 ]; then
-    #   echo "Error with mongodump!"
-    # fi
-
-    ## Redis
-
-    ## Purge previous dumps
-    # rm -rf ${LOCAL_BACKUP_DIR}/redis/
-    # rm -rf ${LOCAL_BACKUP_DIR}/redis-*
-    ## example with copy .rdb file
-    ## for the default instance :
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/redis/
-    # cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/redis/
-    ## for multiple instances :
-    # for instance in $(ls -d /var/lib/redis-*); do
-    #     name=$(basename $instance)
-    #     mkdir -p ${LOCAL_BACKUP_DIR}/${name}
-    #     cp -a ${instance}/dump.rdb ${LOCAL_BACKUP_DIR}/${name}
-    # done
-
-    ## ElasticSearch
-
-    ## Take a snapshot as a backup.
-    ## Warning: You need to have a path.repo configured.
-    ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes
-    # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log
-    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log
-    ## Clustered version here
-    ## It basically the same thing except that you need to check that NFS is mounted
-    # if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2'
-    # then
-    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log
-    #     curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log
-    # else
-    #     echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
-    # fi
-    ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
-    # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
-    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
-    # done
-    # date=$(/bin/date +%F)
-    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log
-
-    ## RabbitMQ
-
-    ## export config
-    #rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> $LOGFILE
-
-    ## MegaCli config
-
-    #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null
-
-    ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls)
-    for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do
-        mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr}
-        traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1
-    done
-
-    server_state_dir="${LOCAL_BACKUP_DIR}/server-state"
-
-    dump_server_state_bin=$(command -v dump-server-state)
-
-    if [ "${SYSTEM}" = "linux" ]; then
-        if [ -n "${dump_server_state_bin}" ]; then
-            ${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
-        else
-            mkdir -p "${server_state_dir}"
-
-            ## Dump system and kernel versions
-            uname -a > ${server_state_dir}/uname.txt
-
-            ## Dump process with ps
-            ps auwwx > ${server_state_dir}/ps.txt
-
-            ## Dump network connections with ss
-            ss -taupen > ${server_state_dir}/netstat.txt
-
-            ## List Debian packages
-            dpkg -l > ${server_state_dir}/packages
-            dpkg --get-selections > ${server_state_dir}/packages.getselections
-            apt-cache dumpavail > ${server_state_dir}/packages.available
-
-            ## Dump iptables
-            if [ -x /sbin/iptables ]; then
-                { /sbin/iptables -L -n -v; /sbin/iptables -t filter -L -n -v; } > ${server_state_dir}/iptables.txt
-            fi
-
-            ## Dump findmnt(8) output
-            FINDMNT_BIN=$(command -v findmnt)
-            if [ -x "${FINDMNT_BIN}" ]; then
-                ${FINDMNT_BIN} > ${server_state_dir}/findmnt.txt
-            fi
-
-            ## Dump MBR / table partitions
-            disks=$(lsblk -l | grep disk | grep -v -E '(drbd|fd[0-9]+)' | awk '{print $1}')
-            for disk in ${disks}; do
-                dd if="/dev/${disk}" of="${server_state_dir}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)"
-                fdisk -l "/dev/${disk}" > "${server_state_dir}/partitions-${disk}" 2>&1
+    ## Verify other evobackup process and kill if needed
+    if [ -e "${PIDFILE}" ]; then
+        pid=$(cat "${PIDFILE}")
+        # Does process still exist ?
+        if kill -0 "${pid}" 2> /dev/null; then
+            # Killing the childs of evobackup.
+            for ppid in $(pgrep -P "${pid}"); do
+                kill -9 "${ppid}";
             done
-            cat ${server_state_dir}/partitions-* > ${server_state_dir}/partitions
-        fi
-    else
-        if [ -n "${dump_server_state_bin}" ]; then
-            ${dump_server_state_bin} --all --force --backup-dir "${server_state_dir}"
+            # Then kill the main PID.
+            kill -9 "${pid}"
+            printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\\n" >&2
         else
-            mkdir -p "${server_state_dir}"
-
-            ## Dump system and kernel versions
-            uname -a > ${server_state_dir}/uname
-
-            ## Dump process with ps
-            ps auwwx > ${server_state_dir}/ps.out
-
-            ## Dump network connections with fstat
-            fstat | head -1 > ${server_state_dir}/netstat.out
-            fstat | grep internet >> ${server_state_dir}/netstat.out
-
-            ## List OpenBSD packages
-            pkg_info -m > ${server_state_dir}/packages
-
-            ## Dump MBR / table partitions
-            disklabel sd0 > ${server_state_dir}/partitions
-
-            ## Dump pf infos
-            pfctl -sa > ${server_state_dir}/pfctl-sa.txt
+            rm -f "${PIDFILE}"
         fi
     fi
+    echo "$$" > "${PIDFILE}"
 
-    ## Dump rights
-    #getfacl -R /var  > ${server_state_dir}/rights-var.txt
-    #getfacl -R /etc  > ${server_state_dir}/rights-etc.txt
-    #getfacl -R /usr  > ${server_state_dir}/rights-usr.txt
-    #getfacl -R /home > ${server_state_dir}/rights-home.txt
+    # Initialize a list of files to delete at exit
+    # Any file added to the list will also be deleted at exit
+    temp_files="${PIDFILE}"
 
-    log "STOP LOCAL_TASKS"
-fi
+    # shellcheck disable=SC2064
+    trap "rm -f ${temp_files}" EXIT
 
-##### REMOTE BACKUP ###################################################
+    # Update canary to keep track of each run
+    update-evobackup-canary --who "${PROGNAME}"
 
-
-if [ "${SYNC_TASKS}" = "1" ]; then
-    n=0
-    server=""
-    if [ "${SERVERS_FALLBACK}" = "1" ]; then
-        # We try to find a suitable server
-        while :; do
-            server=$(pick_server "${n}")
-            test $? = 0 || exit 2
-
-            if test_server "${server}"; then
-                break
-            else
-                server=""
-                n=$(( n + 1 ))
-            fi
-        done
-    else
-        # we force the server
-        server=$(pick_server "${n}")
+    if [ "${LOCAL_TASKS}" = "1" ]; then
+        local_tasks
     fi
 
-    SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
-    SSH_PORT=$(echo "${server}" | cut -d':' -f2)
-
-    if [ "${SYSTEM}" = "linux" ]; then
-        rep="/bin /boot /lib /opt /sbin /usr"
-    else
-        rep="/bsd /bin /sbin /usr"
+    if [ "${SYNC_TASKS}" = "1" ]; then
+        sync_tasks
     fi
 
-    log "START SYNC_TASKS - server=${server}"
+    STOP_EPOCH=$(/bin/date +%s)
 
-    # /!\ DO NOT USE COMMENTS in the rsync command /!\
-    # It breaks the command and destroys data, simply remove (or add) lines.
+    if [ "${SYSTEM}" = "openbsd" ]; then
+        start_time=$(/bin/date -f "%s" -j "${START_EPOCH}" +"${DATE_FORMAT}")
+        stop_time=$(/bin/date -f "%s" -j "${STOP_EPOCH}" +"${DATE_FORMAT}")
+    else
+        start_time=$(/bin/date --date="@${START_EPOCH}" +"${DATE_FORMAT}")
+        stop_time=$(/bin/date --date="@${STOP_EPOCH}" +"${DATE_FORMAT}")
+    fi
+    duration=$(( STOP_EPOCH - START_EPOCH ))
 
-    # Remote shell command
-    RSH_COMMAND="ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'"
+    log "STOP GLOBAL - start='${start_time}' stop='${stop_time}' duration=${duration}s"
 
-    # ignore check because we want it to split the different arguments to $rep
-    # shellcheck disable=SC2086
-    rsync -avzh --relative --stats --delete --delete-excluded --force --ignore-errors --partial \
-        --exclude "dev"                                    \
-        --exclude "lost+found"                             \
-        --exclude ".nfs.*"                                 \
-        --exclude "/usr/doc"                               \
-        --exclude "/usr/obj"                               \
-        --exclude "/usr/share/doc"                         \
-        --exclude "/usr/src"                               \
-        --exclude "/var/apt"                               \
-        --exclude "/var/cache"                             \
-        --exclude "/var/lib/amavis/amavisd.sock"           \
-        --exclude "/var/lib/amavis/tmp"                    \
-        --exclude "/var/lib/clamav/*.tmp"                  \
-        --exclude "/var/lib/elasticsearch"                 \
-        --exclude "/var/lib/metche"                        \
-        --exclude "/var/lib/munin/*tmp*"                   \
-        --exclude "/var/db/munin/*.tmp"                    \
-        --exclude "/var/lib/mysql"                         \
-        --exclude "/var/lib/php5"                          \
-        --exclude "/var/lib/php/sessions"                  \
-        --exclude "/var/lib/postgres"                      \
-        --exclude "/var/lib/postgresql"                    \
-        --exclude "/var/lib/sympa"                         \
-        --exclude "/var/lock"                              \
-        --exclude "/var/run"                               \
-        --exclude "/var/spool/postfix"                     \
-        --exclude "/var/spool/smtpd"                       \
-        --exclude "/var/spool/squid"                       \
-        --exclude "/var/state"                             \
-        --exclude "/var/tmp"                               \
-        --exclude "lxc/*/rootfs/tmp"                       \
-        --exclude "lxc/*/rootfs/usr/doc"                   \
-        --exclude "lxc/*/rootfs/usr/obj"                   \
-        --exclude "lxc/*/rootfs/usr/share/doc"             \
-        --exclude "lxc/*/rootfs/usr/src"                   \
-        --exclude "lxc/*/rootfs/var/apt"                   \
-        --exclude "lxc/*/rootfs/var/cache"                 \
-        --exclude "lxc/*/rootfs/var/lib/php5"              \
-        --exclude "lxc/*/rootfs/var/lib/php/sessions"      \
-        --exclude "lxc/*/rootfs/var/lock"                  \
-        --exclude "lxc/*/rootfs/var/log"                   \
-        --exclude "lxc/*/rootfs/var/run"                   \
-        --exclude "lxc/*/rootfs/var/state"                 \
-        --exclude "lxc/*/rootfs/var/tmp"                   \
-        --exclude "/home/mysqltmp"                         \
-            ${rep}                                         \
-            /etc                                           \
-            /root                                          \
-            /var                                           \
-            /home                                          \
-        -e "${RSH_COMMAND}"                                \
-        "root@${SSH_SERVER}:/var/backup/"                  \
-        | tail -30 >> $LOGFILE
+    tail -20 "${LOGFILE}" | mail -s "[info] EvoBackup - Client ${HOSTNAME}" ${MAIL}
+}
 
-    log "STOP SYNC_TASKS - server=${server}"
-fi
+# set all programs to C language (english)
+export LC_ALL=C
 
-##### REPORTING #######################################################
+# Error on unassigned variable
+set -u
 
-STOP_EPOCH=$(/bin/date +%s)
+# Default return-code (0 == succes)
+rc=0
 
-if [ "${SYSTEM}" = "openbsd" ]; then
-    start_time=$(/bin/date -f "%s" -j "${START_EPOCH}" +"${DATE_FORMAT}")
-    stop_time=$(/bin/date -f "%s" -j "${STOP_EPOCH}" +"${DATE_FORMAT}")
-else
-    start_time=$(/bin/date --date="@${START_EPOCH}" +"${DATE_FORMAT}")
-    stop_time=$(/bin/date --date="@${STOP_EPOCH}" +"${DATE_FORMAT}")
-fi
-duration=$(( STOP_EPOCH - START_EPOCH ))
+# execute main funciton
+main
 
-log "STOP GLOBAL - start='${start_time}' stop='${stop_time}' duration=${duration}s"
-
-tail -20 "${LOGFILE}" \
-    | mail -s "[info] EvoBackup - Client ${HOSTNAME}" ${MAIL}
+exit ${rc}

server/CHANGELOG.md

@@ -6,14 +6,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+### Changed
+
 ### Deprecated
 
 ### Removed
 
 ### Fixed
 
+### Security
+
+## [22.11] - 2022-11-28
+
+### Added
+
+* check-canary: new subcommand to check canary files and content
+
+### Changed
+
+* stats: filter active jails and columnize the output
+
+## [22.07] - 2022-07-20
+
+### Changed
+
+* check-setup: check minifirewall version only if minifirewall is present
+* check-setup: get minifirewall version from internal variable (there is no other backward compatible way)
+* check-setup: use findmnt with mountpoint instead of target
+
+## [22.06] - 2022-06-28
+
+### Added
+
+* bkctld-init: create "incs/\<jail\>" directory for jails
+
+### Fixed
+
 * shell syntax error when ${btrfs_bin} variable is empty
 * read_variable + read_numerical_variable: keep the last found value only
+* Debian 8 findmnt(8) support
 
 ### Security
 

server/README.md

@@ -4,7 +4,7 @@ Bkctld (aka server-side evobackup)
 bkctld helps you manage the receiving side of a backup infrastructure.
 It is licensed under the AGPLv3.
 
-With bkctld you create and manage "jails". They contain a chrooted and dedicated SSH server, with it's own TCP port and optionnaly it's own set of iptables rules.
+With bkctld you create and manage "jails". They contain a chrooted and dedicated SSH server, with its own TCP port and optionally its own set of iptables rules.
 
 With bkctld you can have hundreds of jails, one for each client to push its data (using Rsync/SFTP). Each client can only see its own data.
 
@@ -30,9 +30,7 @@ This volume can also be encrypted with **LUKS**.
 
 ## Security considerations
 
-The client obviously has access to its uploaded data (in the chroot), but the timestamped copies are outside the chroot, to reduce the risk or complete backup erasure from a compromised client.
-
-Since the client connects to the backup server with root, it can mess with the jail and destroy the data. But the timestamped copies are out of reach because outside of the chroot.
+The client obviously has access to its uploaded data (in the chroot), but the timestamped copies are outside the chroot, to reduce the risk of complete backup erasure from a compromised client.
 
 It means that **if the client server is compromised**, an attacker can destroy the latest copy of the backed up data, but not the timestamped copies.
 And **if the backup server is compromised** an attacker has complete access to all the backup data (inside and outside the jails), but they don't have any access to the client.
@@ -41,7 +39,7 @@ This architecture is as secure as SSH, Rsync, chroot and iptables are.
 
 ## Install
 
-See the [installation guide](docs/install.md) for instructions.
+See the [installation guide](https://intra.evolix.net/OutilsInternes/bkctld) for instructions.
 
 ## Testing
 
@@ -51,6 +49,12 @@ You can deploy test environments with Vagrant :
 vagrant up
 ~~~
 
+To destroy Vagrant VMs :
+
+~~~
+vagrant destroy
+~~~
+
 ### Deployment
 
 Run `vagrant rsync-auto` in a terminal for automatic synchronization of
@@ -77,6 +81,8 @@ vagrant@buster-btrfs $ sudo -i
 root@buster-btrfs # bats /vagrant/test/*.bats
 ~~~
 
+[comment]: <> (* pour vim)
+
 You should shellcheck your bats files, but with shellcheck > 0.4.6, because the 0.4.0 version doesn't support bats syntax.
 
 ## Usage
@@ -99,7 +105,7 @@ pandoc -f markdown \
 #### Client configuration
 
 You can backup various systems in the evobackup jails : Linux, BSD,
-Windows, macOS. The only need Rsync or an SFTP client.
+Windows, macOS. The only need is Rsync or an SFTP client.
 
 ~~~
 rsync -av -e "ssh -p SSH_PORT" /home/ root@SERVER_NAME:/var/backup/home/

server/bkctld

@@ -53,6 +53,9 @@ while :; do
         -f|--force)
             export FORCE=1
             ;;
+        --no-header)
+            export HEADER=0
+            ;;
         *)
             # Default case: If no more options then break out of the loop.
             break
@@ -64,7 +67,7 @@ done
 
 subcommand="${1:-}"
 case "${subcommand}" in
-    "inc" | "rm" | "check-jails" | "check-setup" | "stats" | "list")
+    "inc" | "rm" | "check-jails" | "check-setup" | "check-canary" | "stats" | "list")
         "${LIBDIR}/bkctld-${subcommand}"
     ;;
     "check")
@@ -116,7 +119,9 @@ case "${subcommand}" in
     ;;
     "status")
         jail_name="${2:-}"
-        printf '%-30s %-10s %-10s %-25s %-20s\n' 'JAIL NAME' 'STATUS' 'PORT' 'RETENTION (DAY/MONTH)' 'IP'
+        if [ "${HEADER}" = "1" ]; then
+            printf '%-30s %-10s %-10s %-25s %-20s\n' 'JAIL NAME' 'STATUS' 'PORT' 'RETENTION (DAY/MONTH)' 'IP'
+        fi
         if [ "${jail_name}" = "all" ] || [ -z "${jail_name}" ]; then
             for jail in $("${LIBDIR}/bkctld-list"); do
                 "${LIBDIR}/bkctld-${subcommand}" "${jail}"

server/bkctld.conf

@@ -15,3 +15,5 @@
 #LOGLEVEL=6
 #NODE=''
 #ARCHIVESDIR='/backup/archives'
+#WARNING=48
+#CRITICAL=72

server/docs/install.md

@@ -2,15 +2,7 @@
 
 ## Install from package
 
-A Debian package is available in the Evolix repository
-
-~~~
-echo "deb http://pub.evolix.net/ stretch" >> /etc/apt/sources.list
-apt update
-apt install bkctld
-~~~
-
-Then edit `/etc/default/bkctld`
+The install documentation is [here](https://intra.evolix.net/OutilsInternes/bkctld)
 
 ## Instal from sources
 
@@ -19,17 +11,17 @@ Warning: `cp`-ing the files without `-n` or `-i` will replace existing files !
 ~~~
 # git clone https://gitea.evolix.org/evolix/evobackup.git
 # cd evobackup
-# cp bkctld /usr/local/sbin/
+# cp server/bkctld /usr/local/sbin/
 # mkdir -p /usr/local/lib/bkctld
-# cp lib/* /usr/local/lib/bkctld/
+# cp server/lib/* /usr/local/lib/bkctld/
 # mkdir -p /usr/local/share/bkctld
-# cp tpl/* /usr/local/share/bkctld/
-# cp bkctld.service /lib/systemd/system/
+# cp server/tpl/* /usr/local/share/bkctld/
+# cp server/bkctld.service /lib/systemd/system/
 # mkdir -p /usr/local/share/doc/bkctld
-# cp zzz_evobackup /usr/local/share/doc/bkctld/
+# cp client/zzz_evobackup /usr/local/share/doc/bkctld/
 # mkdir -p /usr/local/share/bash_completion/
-# cp bash_completion /usr/local/share/bash_completion/bkctld
-# cp bkctld.conf /etc/default/bkctld
+# cp server/bash_completion /usr/local/share/bash_completion/bkctld
+# cp server/bkctld.conf /etc/default/bkctld
 ~~~
 
 ## Chroot dependencies

server/lib/bkctld-check-canary

@@ -0,0 +1,59 @@
+#!/bin/sh
+#
+# Description: check canary file
+# Usage: check-canary [<jailname>|all]
+#
+
+# shellcheck source=./includes
+LIBDIR="$(dirname $0)" && . "${LIBDIR}/includes"
+
+return=0
+nb_crit=0
+nb_warn=0
+nb_ok=0
+nb_unkn=0
+output=""
+
+date=$(date +"%Y-%m-%d")
+
+# Check each jail status
+
+check_jail() {
+    jail_name=$1
+
+    jail_path=$(jail_path "${jail_name}")
+    canary_absolute_file="${jail_path}/var/backup/${CANARY_RELATIVE_FILE}"
+
+    if [ -f "${canary_absolute_file}" ]; then
+        if grep --quiet --fixed-string "${date}" "${canary_absolute_file}"; then
+            nb_ok=$((nb_ok + 1))
+            output="${output}OK - ${jail_name} - entries found for ${date} in ${CANARY_RELATIVE_FILE} file\n"
+        else
+            nb_crit=$((nb_crit + 1))
+            output="${output}CRITICAL - ${jail_name} - No entry for ${date} in ${CANARY_RELATIVE_FILE} file\n"
+            [ "${return}" -le 2 ] && return=2
+        fi
+    else
+        nb_crit=$((nb_crit + 1))
+        output="${output}CRITICAL - ${jail_name} - missing ${CANARY_RELATIVE_FILE} file\n"
+        [ "${return}" -le 2 ] && return=2
+    fi
+}
+
+for jail_name in $(jails_list); do
+    check_jail "${jail_name}"
+done
+
+[ "${return}" -ge 0 ] && header="OK"
+[ "${return}" -ge 1 ] && header="WARNING"
+[ "${return}" -ge 2 ] && header="CRITICAL"
+[ "${return}" -ge 3 ] && header="UNKNOWN"
+
+printf "%s - %s UNK / %s CRIT / %s WARN / %s OK\n\n" "${header}" "${nb_unkn}" "${nb_crit}" "${nb_warn}" "${nb_ok}"
+
+printf "${output}" | grep -E "^UNKNOWN"
+printf "${output}" | grep -E "^CRITICAL"
+printf "${output}" | grep -E "^WARNING"
+printf "${output}" | grep -E "^OK"
+
+exit "${return}"

server/lib/bkctld-check-setup

@@ -16,7 +16,7 @@ output=""
 
 # Verify backup partition is mounted and writable
 
-findmnt --mountpoint "${BACKUP_PARTITION}" -O rw > /dev/null
+findmnt -O rw --mountpoint "${BACKUP_PARTITION}" > /dev/null
 if [ "$?" -ne 0 ]; then
     nb_crit=$((nb_crit + 1))
     output="${output}CRITICAL - Backup disk \`/backup' is not mounted (or read-only) !\n"
@@ -29,11 +29,12 @@ fi
 # Check if the firewall file is sourced
 
 minifirewall_config=/etc/default/minifirewall
-minifirewall_version=$(/etc/init.d/minifirewall status | head -1 | cut -d ' ' -f 3)
 
 if [ -n "${FIREWALL_RULES}" ] \
 && [ -r "${FIREWALL_RULES}" ] \
 && [ -f "${minifirewall_config}" ]; then
+    minifirewall_version=$(grep -E -o "^VERSION=(\S+)" /etc/init.d/minifirewall | head -1 | cut -d '=' -f 2 | tr -d "'" | tr -d '"')
+
     if [ -n "${minifirewall_version}" ] && dpkg --compare-versions "${minifirewall_version}" ge "22.03"; then
         # Minifirewall 22.03+ includes files automatically
         nb_ok=$((nb_ok + 1))

server/lib/bkctld-init

@@ -12,11 +12,12 @@ if [ -z "${jail_name}" ]; then
     show_help && exit 1
 fi
 jail_path=$(jail_path "${jail_name}")
+incs_path=$(incs_path "${jail_name}")
 
 test -d "${jail_path}" && error "Skip jail \`${jail_name}' : it already exists"
 
-# Create config and jails directory
-mkdir --parents "${CONFDIR}" "${JAILDIR}"
+# Create config, jails and incs directories
+mkdir --parents "${CONFDIR}" "${JAILDIR}" "${INCDIR}"
 
 if is_btrfs "$(dirname "${JAILDIR}")" || is_btrfs "${JAILDIR}"; then
     btrfs_bin=$(command -v btrfs)
@@ -28,6 +29,8 @@ else
     mkdir --parents "${jail_path}"
 fi
 
+mkdir --parents "${incs_path}"
+
 setup_jail_chroot "${jail_name}"
 setup_jail_config "${jail_name}"
 

server/lib/bkctld-rename

@@ -51,15 +51,15 @@ if dry_run; then
 else
     mv "${jail_path}" "${new_jail_path}"
 fi
-if dry_run; then
-    if [ -d "${incs_path}" ]; then
+
+if [ -d "${incs_path}" ]; then
+    if dry_run; then
         echo "[dry-run] rename ${incs_path} to ${new_incs_path}"
-    fi
-else
-    if [ -d "${incs_path}" ]; then
+    else
         mv "${incs_path}" "${new_incs_path}"
     fi
 fi
+
 if [ -d "${jail_config_dir}" ]; then
     if dry_run; then
         echo "[dry-run] rename ${jail_config_dir} to ${new_jail_config_dir}"

server/lib/bkctld-stats

@@ -15,27 +15,29 @@ ionice -c3 "${DUC}" index -d "${IDX_FILE}" "${JAILDIR}"
 touch "${INDEX_DIR}/.lastrun.duc"
 EOF
 
-[ ! -f "${INDEX_DIR}/.lastrun.duc" ] && notice "First run of DUC always in progress ..." && exit 0
+[ ! -f "${INDEX_DIR}/.lastrun.duc" ] && notice "First run of DUC still in progress ..." && exit 0
 [ ! -f ${IDX_FILE} ] && error "Index file doesn't exits !"
 
 printf "Last update of index file : "
 stat --format=%Y "${INDEX_DIR}/.lastrun.duc" | xargs -i -n1 date -R -d "@{}"
-echo "<jail> <size> <incs> <lastconn>" | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $1, $2, $3, $4); }'
 
 duc_output=$(mktemp)
 stat_output=$(mktemp)
 incs_output=$(mktemp)
+jail_patterns_list=$(mktemp)
 
 # shellcheck disable=SC2064
-trap "rm ${duc_output} ${incs_output} ${stat_output}" 0
+trap "rm ${duc_output} ${incs_output} ${stat_output} ${jail_patterns_list}" 0
 
-"${DUC}" ls -d "${IDX_FILE}" "${JAILDIR}" > "${duc_output}"
+"${DUC}" ls --database "${IDX_FILE}" "${JAILDIR}" > "${duc_output}"
 
-awk '{ print $2 }' "${duc_output}" | while read jail_name; do
+jails_list | sed -e "s/^\(.*\)$/\\\\b\1\\\\b/" > "${jail_patterns_list}"
+
+grep -f "${jail_patterns_list}" "${duc_output}" | awk '{ print $2 }' | while read jail_name; do
     jail_path=$(jail_path "${jail_name}")
     stat --format=%Y "${jail_path}/var/log/lastlog" | xargs -i -n1 date -d "@{}" "+%d-%m-%Y" >> "${stat_output}"
 
-    incs_policy_file=$(current_jail_incs_policy_file ${jail_name})
+    incs_policy_file=$(current_jail_incs_policy_file "${jail_name}")
     incs_policy="0"
     if [ -r "${incs_policy_file}" ]; then
         days=$(grep "^\+" "${incs_policy_file}" | grep --count "day")
@@ -45,4 +47,7 @@ awk '{ print $2 }' "${duc_output}" | while read jail_name; do
     echo "${incs_policy}" >> "${incs_output}"
 done
 
-paste "${duc_output}" "${incs_output}" "${stat_output}" | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $2, $1, $3, $4); }'
+(
+    echo "<jail> <size> <incs> <lastconn>"
+    paste "${duc_output}" "${incs_output}" "${stat_output}" | awk '{ printf("%s %s %s %s\n", $2, $1, $3, $4); }'
+) | column -t

server/lib/bkctld-status

@@ -1,7 +1,7 @@
 #!/bin/sh
 #
 # Description: Display status of SSH server
-# Usage: status [<jailname>|all]
+# Usage: [--no-header] status [<jailname>|all]
 #
 
 # shellcheck source=./includes

server/lib/includes

@@ -6,7 +6,7 @@
 
 [ -f /etc/default/bkctld ] && . /etc/default/bkctld
 
-VERSION="22.04"
+VERSION="22.11"
 
 LIBDIR=${LIBDIR:-/usr/lib/bkctld}
 CONFDIR="${CONFDIR:-/etc/evobackup}"
@@ -20,6 +20,7 @@ LOCKDIR="${LOCKDIR:-/run/lock/bkctld}"
 ARCHIVESDIR="${ARCHIVESDIR:-${BACKUP_PARTITION}/archives}"
 INDEX_DIR="${INDEX_DIR:-${BACKUP_PARTITION}/index}"
 IDX_FILE="${IDX_FILE:-${INDEX_DIR}/bkctld-jails.idx}"
+CANARY_RELATIVE_FILE="${CANARY_RELATIVE_FILE:-/zzz_evobackup_canary}"
 SSHD_PID="${SSHD_PID:-/run/sshd.pid}"
 SSHD_CONFIG="${SSHD_CONFIG:-/etc/ssh/sshd_config}"
 AUTHORIZED_KEYS="${AUTHORIZED_KEYS:-/root/.ssh/authorized_keys}"
@@ -29,6 +30,7 @@ CRITICAL="${CRITICAL:-48}"
 WARNING="${WARNING:-24}"
 DUC=$(command -v duc-nox || command -v duc)
 FORCE="${FORCE:-0}"
+HEADER="${HEADER:-1}"
 
 show_version() {
     cat <<END
@@ -62,6 +64,13 @@ EOF
     printf "\n"
 }
 
+is_quiet() {
+    test ${QUIET} -eq 1
+}
+is_verbose() {
+    test ${VERBOSE} -eq 1
+}
+
 log_date() {
     echo "[$(date +"%Y-%m-%d %H:%M:%S")]"
 }
@@ -127,7 +136,7 @@ is_btrfs() {
 
     inode=$(stat --format=%i "${path}")
 
-    test $inode -eq 256
+    test "$inode" -eq 256
 }
 
 # Returns the list of jails found in the "jails" directory (default)

server/test/checks.bats

@@ -144,7 +144,7 @@ OUT
 @test "Check setup WARNING if firewall rules are not sourced" {
     /usr/lib/bkctld/bkctld-start ${JAILNAME}
 
-    mkdir /etc/minifirewall.d/
+    mkdir --parents /etc/minifirewall.d/
     firewall_rules_file="/etc/minifirewall.d/bkctld"
     set_variable "/etc/default/bkctld" "FIREWALL_RULES" "${firewall_rules_file}"
     echo "" > "${firewall_rules_file}"
@@ -159,7 +159,7 @@ OUT
 @test "Check setup OK if firewall rules are sourced" {
     /usr/lib/bkctld/bkctld-start ${JAILNAME}
 
-    mkdir /etc/minifirewall.d/
+    mkdir --parents /etc/minifirewall.d/
     firewall_rules_file="/etc/minifirewall.d/bkctld"
     set_variable "/etc/default/bkctld" "FIREWALL_RULES" "${firewall_rules_file}"
     echo "" > "${firewall_rules_file}"
@@ -252,3 +252,25 @@ OUT
     assert_failure
 }
 # TODO: write many more tests for bkctld-check-incs
+
+@test "Check-canary fails if a canary file doesn't exist" {
+    run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}"
+    assert_equal "$status" "2"
+    assert_line "CRITICAL - ${JAILNAME} - missing /zzz_evobackup_canary file"
+}
+
+@test "Check-canary fails if a canary is missing today's entries" {
+    today="$(date +%Y-%m-%d)"
+    touch "${JAILPATH}/var/backup/zzz_evobackup_canary"
+
+    run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}"
+    assert_equal "$status" "2"
+    assert_line "CRITICAL - ${JAILNAME} - No entry for ${today} in /zzz_evobackup_canary file"
+}
+
+@test "Check-canary succeeds if a canary has today's entries" {
+    echo "$(date "+%FT%T%z") bats-test" >> "${JAILPATH}/var/backup/zzz_evobackup_canary"
+
+    run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}"
+    assert_success
+}

server/test/incs.bats

@@ -9,6 +9,11 @@ load test_helper
     run test -e "${CONFDIR}/${JAILNAME}.d/incs_policy"
     [ "${status}" -eq 0 ]
 }
+@test "Inc directory after jail init" {
+    # An incs_policy file should exist
+    run test -d "${INCDIR}/${JAILNAME}"
+    [ "${status}" -eq 0 ]
+}
 
 @test "Normal inc creation" {
     /usr/lib/bkctld/bkctld-inc