client: Release 22.12

* use a log file for rsync
* build the command argument by argument, without backslashes
* move excludes into a file

Co-authored-by: Jeremy Lecour <jlecour@evolix.fr>
Reviewed-on: #63

.Jenkinsfile

@@ -1,24 +1,15 @@
 pipeline {
-    agent { label 'docker' }
+    agent { label 'sbuild' }
     stages {
         stage('Build Debian package') {
-            agent {
-                docker {
-                    image 'evolix/gbp:bullseye'
-                    args '-u root --privileged'
-                }
-            }
             when {
                 branch 'debian'
             }
             steps {
                 script {
-                    sh 'mk-build-deps --install --remove debian/control'
-                    sh 'rm -rf {source,*.gz,*.bz2,*.xz,*.deb,*.dsc,*.changes,*.buildinfo,lintian.txt,.git}'
-                    sh "gbp clone --debian-branch=$GIT_BRANCH $GIT_URL source"
-                    sh 'cd source && git checkout $GIT_BRANCH && gbp buildpackage -us -uc'
+                    sh 'gbp buildpackage'
                 }
-                archiveArtifacts allowEmptyArchive: true, artifacts: '*.gz,*.bz2,*.xz,*.deb,*.dsc,*.changes,*.buildinfo,lintian.txt'
+                archiveArtifacts allowEmptyArchive: true, artifacts: 'build-area/*.gz,build-area/*.bz2,build-area/*.xz,build-area/*.deb,build-area/*.dsc,build-area/*.changes,build-area/*.buildinfo,build-area/*.build,build-area/lintian.txt'
             }
         }
 
@@ -28,10 +19,7 @@ pipeline {
             }
             steps {
                 script {
-                    sh 'echo Dummy line to remove once something actually happens.'
-                 /* No crendentials yet.
-                    sh 'rsync -avP bkctld* droneci@pub.evolix.net:/home/droneci/bkctld/'
-                  */
+                    sh 'rsync -avP build-area/bkctld*.deb build-area/bkctld*.changes build-area/bkctld*.buildinfo pub.evolix.org:/srv/upload/'
                 }
             }
         }

.drone.yml

@@ -1,35 +0,0 @@
-kind: pipeline
-name: default
-
-steps:
-- name: fetch
-  image: alpine/git
-  commands:
-  - git fetch --tags
-
-- name: build debian package
-  image: evolix/gbp:bullseye
-  branches:
-  - debian
-  commands:
-  - mk-build-deps --install --remove debian/control
-  - git clean --force
-  - gbp buildpackage -us -uc
-  when:
-    branch:
-    - debian
-
-- name: upload debian package
-  image: drillster/drone-rsync
-  settings:
-    hosts: ["pub.evolix.net"]
-    port: 22
-    user: droneci
-    key:
-      from_secret: drone_private_key
-    target: /home/droneci/bkctld/
-    source: ../bkctld*
-    delete: true
-  when:
-    branch:
-    - debian

.gitignore

@@ -0,0 +1,2 @@
+*.swp
+.vagrant

client/CHANGELOG.md

@@ -10,6 +10,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+### Deprecated
+
+### Removed
+
+### Fixed
+
+### Security
+
+## [22.12]
+### Changed
+
 * Use --dump-dir instead of --backup-dir to suppress dump-server-state warning
 * Do not use rsync compression
 * Replace rsync option --verbose by --itemize-changes
@@ -17,17 +28,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * update-evobackup-canary: do not use GNU date, for it to be compatible with OpenBSD
 * Add AGPL License and README
 * Script now depends on Bash
-
-### Deprecated
+* tolerate absence of mtr or traceroute
+* Only one loop for all Redis instances
+* remodel how we build the rsync command
+* use sub shells instead of moving around
+* Separate Rsync for the canary file if the main Rsync has finished without errors
 
 ### Removed
 
+* No more fallback if dump-server-state is missing
+
 ### Fixed
 
 * Make start_time and stop_time compatible with OpenBSD
 
-### Security
-
 ## [22.03]
 
 Split client and server parts of the project

client/zzz_evobackup

@@ -18,7 +18,7 @@
 
 ##### Configuration ###################################################
 
-VERSION="22.05"
+VERSION="22.12"
 
 # email adress for notifications
 MAIL=jdoe@example.com
@@ -48,6 +48,9 @@ PIDFILE="/var/run/${PROGNAME}.pid"
 # Customize the log path if you have multiple scripts and with separate logs
 LOGFILE="/var/log/evobackup.log"
 
+# Full Rsync log file, reset each time
+RSYNC_LOGFILE="/var/log/${PROGNAME}.rsync.log"
+
 HOSTNAME=$(hostname)
 
 DATE_FORMAT="%Y-%m-%d %H:%M:%S"
@@ -57,7 +60,72 @@ DATE_FORMAT="%Y-%m-%d %H:%M:%S"
 # Enable/disable sync tasks (default: enabled)
 : "${SYNC_TASKS:=1}"
 
-##### SETUP AND FUNCTIONS #############################################
+CANARY_FILE="/zzz_evobackup_canary"
+
+# Source paths can be customized
+# Empty lines, and lines containing # or ; are ignored
+RSYNC_INCLUDES="
+/etc
+/root
+/var
+/home
+"
+
+# Excluded paths can be customized
+# Empty lines, and lines beginning with # or ; are ignored
+RSYNC_EXCLUDES="
+/dev
+/proc
+/run
+/sys
+/tmp
+/usr/doc
+/usr/obj
+/usr/share/doc
+/usr/src
+/var/apt
+/var/cache
+/var/db/munin/*.tmp
+/var/lib/amavis/amavisd.sock
+/var/lib/amavis/tmp
+/var/lib/clamav/*.tmp
+/var/lib/elasticsearch
+/var/lib/metche
+/var/lib/mongodb
+/var/lib/munin/*tmp*
+/var/lib/mysql
+/var/lib/php/sessions
+/var/lib/php5
+/var/lib/postgres
+/var/lib/postgresql
+/var/lib/sympa
+/var/lock
+/var/run
+/var/spool/postfix
+/var/spool/smtpd
+/var/spool/squid
+/var/state
+/var/tmp
+lost+found
+.nfs.*
+lxc/*/rootfs/tmp
+lxc/*/rootfs/usr/doc
+lxc/*/rootfs/usr/obj
+lxc/*/rootfs/usr/share/doc
+lxc/*/rootfs/usr/src
+lxc/*/rootfs/var/apt
+lxc/*/rootfs/var/cache
+lxc/*/rootfs/var/lib/php5
+lxc/*/rootfs/var/lib/php/sessions
+lxc/*/rootfs/var/lock
+lxc/*/rootfs/var/run
+lxc/*/rootfs/var/state
+lxc/*/rootfs/var/tmp
+/home/mysqltmp
+"
+
+
+##### FUNCTIONS #######################################################
 
 local_tasks() {
     log "START LOCAL_TASKS"
@@ -76,7 +144,7 @@ local_tasks() {
     # rm -rf ${LOCAL_BACKUP_DIR}/mysql
     # rm -rf ${LOCAL_BACKUP_DIR}/mysqlhotcopy
     # rm -rf /home/mysqldump
-    # rm -f ${LOCAL_BACKUP_DIR}/*.err ${LOCAL_BACKUP_DIR}/**/*.err
+    # find ${LOCAL_BACKUP_DIR}/ -type f -name '*.err' -delete
 
     ## example with global and compressed mysqldump
     # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
@@ -90,7 +158,7 @@ local_tasks() {
     ## example with compressed SQL dump (with data) for each databases
     # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
     # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
-    # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
+    # | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)"); do
     #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i 2> ${LOCAL_BACKUP_DIR}/${i}.err | gzip --best > ${LOCAL_BACKUP_DIR}/mysql/${i}.sql.gz
     #     last_rc=$?
     #     if [ ${last_rc} -ne 0 ]; then
@@ -119,7 +187,7 @@ local_tasks() {
     ## example with SQL dump (schema only, no data) for each databases
     # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
     # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
-    # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
+    # | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)"); do
     #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases $i 2> ${LOCAL_BACKUP_DIR}/${i}.schema.err > ${LOCAL_BACKUP_DIR}/mysql/${i}.schema.sql
     #     last_rc=$?
     #     if [ ${last_rc} -ne 0 ]; then
@@ -141,7 +209,7 @@ local_tasks() {
 
     ## example with two dumps for each table (.sql/.txt) for all databases
     # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
-    # | egrep -v "^(Database|information_schema|performance_schema|sys)" ); do
+    # | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)" ); do
     #     mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump
     #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \
     #  --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i 2> /home/mysqldump/$i.err"
@@ -163,7 +231,7 @@ local_tasks() {
 
     ## example for multiples MySQL instances
     # mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3)
-    # grep -E "^port\s*=\s*\d*" /etc/mysql/my.cnf |while read instance; do
+    # grep --extended-regexp "^port\s*=\s*\d*" /etc/mysql/my.cnf | while read instance; do
     #     instance=$(echo "$instance"|awk '{ print $3 }')
     #     if [ "$instance" != "3306" ]
     #     then
@@ -182,13 +250,16 @@ local_tasks() {
     # rm -rf ${LOCAL_BACKUP_DIR}/pg.*.gz
     # rm -rf ${LOCAL_BACKUP_DIR}/pg-backup.tar
     # rm -rf ${LOCAL_BACKUP_DIR}/postgresql/*
+
     ## example with pg_dumpall (warning: you need space in ~postgres)
     # su - postgres -c "pg_dumpall > ~/pg.dump.bak"
     # mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/
+
     ## another method with gzip directly piped
-    # cd /var/lib/postgresql
-    # sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
-    # cd - > /dev/null
+    # (
+    #    cd /var/lib/postgresql;
+    #    sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
+    # )
 
     ## example with all tables from MYBASE excepts TABLE1 and TABLE2
     # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE
@@ -199,9 +270,11 @@ local_tasks() {
     ## example with compressed PostgreSQL dump for each databases
     # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/postgresql
     # chown postgres:postgres ${LOCAL_BACKUP_DIR}/postgresql
-    # dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*)
-    # 
-    # for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done
+    # (
+    #    cd /var/lib/postgresql
+    #    dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*)
+    #    for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done
+    # )
 
     ## MongoDB
 
@@ -221,15 +294,13 @@ local_tasks() {
     ## Purge previous dumps
     # rm -rf ${LOCAL_BACKUP_DIR}/redis/
     # rm -rf ${LOCAL_BACKUP_DIR}/redis-*
-    ## example with copy .rdb file
-    ## for the default instance :
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/redis/
-    # cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/redis/
-    ## for multiple instances :
-    # for instance in $(ls -d /var/lib/redis-*); do
-    #     name=$(basename $instance)
-    #     mkdir -p ${LOCAL_BACKUP_DIR}/${name}
-    #     cp -a ${instance}/dump.rdb ${LOCAL_BACKUP_DIR}/${name}
+    ## Copy dump.rdb file for each found instance
+    # for instance in $(find /var/lib/ -mindepth 1 -maxdepth 1 -type d -name 'redis*'); do
+    #     if [ -f "${instance}/dump.rdb" ]; then
+    #         name=$(basename $instance)
+    #         mkdir -p ${LOCAL_BACKUP_DIR}/${name}
+    #         cp -a "${instance}/dump.rdb" "${LOCAL_BACKUP_DIR}/${name}"
+    #     fi
     # done
 
     ## ElasticSearch
@@ -248,7 +319,7 @@ local_tasks() {
     # else
     #     echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
     # fi
-    ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
+    ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
     # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
     #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
     # done
@@ -258,24 +329,35 @@ local_tasks() {
     ## RabbitMQ
 
     ## export config
-    #rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> "${LOGFILE}"
+    # rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> "${LOGFILE}"
 
     ## MegaCli config
 
-    #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null
+    # megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null
 
     ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls)
-    for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do
-        mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr}
-        traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1
-    done
+    network_targets="8.8.8.8 www.evolix.fr travaux.evolix.net"
+    mtr_bin=$(command -v mtr)
+    if [ -n "${mtr_bin}" ]; then
+        for addr in ${network_targets}; do
+            ${mtr_bin} -r "${addr}" > "${LOCAL_BACKUP_DIR}/mtr-${addr}"
+        done
+    fi
+    traceroute_bin=$(command -v traceroute)
+    if [ -n "${traceroute_bin}" ]; then
+        for addr in ${network_targets}; do
+            ${traceroute_bin} -n "${addr}" > "${LOCAL_BACKUP_DIR}/traceroute-${addr}" 2>&1
+        done
+    fi
 
     server_state_dir="${LOCAL_BACKUP_DIR}/server-state"
 
     dump_server_state_bin=$(command -v dump-server-state)
-
-    if [ "${SYSTEM}" = "linux" ]; then
-        if [ -n "${dump_server_state_bin}" ]; then
+    if [ -z "${dump_server_state_bin}" ]; then
+        error "dump-server-state is missing"
+        rc=1
+    else
+        if [ "${SYSTEM}" = "linux" ]; then
             ${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
             last_rc=$?
             if [ ${last_rc} -ne 0 ]; then
@@ -283,81 +365,92 @@ local_tasks() {
                 rc=1
             fi
         else
-            mkdir -p "${server_state_dir}"
-
-            ## Dump system and kernel versions
-            uname -a > ${server_state_dir}/uname.txt
-
-            ## Dump process with ps
-            ps auwwx > ${server_state_dir}/ps.txt
-
-            ## Dump network connections with ss
-            ss -taupen > ${server_state_dir}/netstat.txt
-
-            ## List Debian packages
-            dpkg -l > ${server_state_dir}/packages
-            dpkg --get-selections > ${server_state_dir}/packages.getselections
-            apt-cache dumpavail > ${server_state_dir}/packages.available
-
-            ## Dump iptables
-            if [ -x /sbin/iptables ]; then
-                { /sbin/iptables -L -n -v; /sbin/iptables -t filter -L -n -v; } > ${server_state_dir}/iptables.txt
-            fi
-
-            ## Dump findmnt(8) output
-            FINDMNT_BIN=$(command -v findmnt)
-            if [ -x "${FINDMNT_BIN}" ]; then
-                ${FINDMNT_BIN} > ${server_state_dir}/findmnt.txt
-            fi
-
-            ## Dump MBR / table partitions
-            disks=$(lsblk -l | grep disk | grep -v -E '(drbd|fd[0-9]+)' | awk '{print $1}')
-            for disk in ${disks}; do
-                dd if="/dev/${disk}" of="${server_state_dir}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)"
-                fdisk -l "/dev/${disk}" > "${server_state_dir}/partitions-${disk}" 2>&1
-            done
-            cat ${server_state_dir}/partitions-* > ${server_state_dir}/partitions
-        fi
-    else
-        if [ -n "${dump_server_state_bin}" ]; then
             ${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
             last_rc=$?
             if [ ${last_rc} -ne 0 ]; then
                 error "dump-server-state returned an error ${last_rc}, check ${server_state_dir}"
                 rc=1
             fi
-        else
-            mkdir -p "${server_state_dir}"
-
-            ## Dump system and kernel versions
-            uname -a > ${server_state_dir}/uname
-
-            ## Dump process with ps
-            ps auwwx > ${server_state_dir}/ps.out
-
-            ## Dump network connections with fstat
-            fstat | head -1 > ${server_state_dir}/netstat.out
-            fstat | grep internet >> ${server_state_dir}/netstat.out
-
-            ## List OpenBSD packages
-            pkg_info -m > ${server_state_dir}/packages
-
-            ## Dump MBR / table partitions
-            disklabel sd0 > ${server_state_dir}/partitions
-
-            ## Dump pf infos
-            pfctl -sa > ${server_state_dir}/pfctl-sa.txt
         fi
     fi
 
     ## Dump rights
-    #getfacl -R /var  > ${server_state_dir}/rights-var.txt
-    #getfacl -R /etc  > ${server_state_dir}/rights-etc.txt
-    #getfacl -R /usr  > ${server_state_dir}/rights-usr.txt
-    #getfacl -R /home > ${server_state_dir}/rights-home.txt
+    # getfacl -R /var  > ${server_state_dir}/rights-var.txt
+    # getfacl -R /etc  > ${server_state_dir}/rights-etc.txt
+    # getfacl -R /usr  > ${server_state_dir}/rights-usr.txt
+    # getfacl -R /home > ${server_state_dir}/rights-home.txt
 
     log "STOP LOCAL_TASKS"
 }
+build_rsync_main_cmd() {
+    ###################################################################
+    # /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ #
+    ###################################################################
+    # DO NOT USE COMMENTS in rsync lines                              #
+    # DO NOT ADD WHITESPACES AFTER \ in rsync lines                   #
+    # It breaks the command and destroys data                         #
+    # You should not modify this, unless you are really REALLY sure   #
+    ###################################################################
+
+    # Create a temp file for excludes and includes
+    includes_file="$(mktemp "${PROGNAME}.includes.XXXXXX")"
+    excludes_file="$(mktemp "${PROGNAME}.excludes.XXXXXX")"
+    # … and add them to the list of files to delete at exit
+    temp_files="${temp_files} ${includes_file} ${excludes_file}"
+
+    # Store includes/excludes in files
+    # without blank lines of comments (# or ;)
+    echo "${RSYNC_INCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${includes_file}"
+    echo "${RSYNC_EXCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${excludes_file}"
+
+    # Rsync command
+    cmd="$(command -v rsync)"
+
+    # Rsync main options
+    cmd="${cmd} --archive"
+    cmd="${cmd} --itemize-changes"
+    cmd="${cmd} --quiet"
+    cmd="${cmd} --stats"
+    cmd="${cmd} --human-readable"
+    cmd="${cmd} --relative"
+    cmd="${cmd} --partial"
+    cmd="${cmd} --delete"
+    cmd="${cmd} --delete-excluded"
+    cmd="${cmd} --force"
+    cmd="${cmd} --ignore-errors"
+    cmd="${cmd} --log-file=${RSYNC_LOGFILE}"
+    cmd="${cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'"
+
+    # Rsync excludes
+    while read line ; do
+        cmd="${cmd} --exclude ${line}"
+    done < "${excludes_file}"
+
+    # Rsync local sources
+    cmd="${cmd} ${default_includes}"
+    while read line ; do
+        cmd="${cmd} ${line}"
+    done < "${includes_file}"
+
+    # Rsync remote destination
+    cmd="${cmd} root@${SSH_SERVER}:/var/backup/"
+
+    # output final command
+    echo "${cmd}"
+}
+build_rsync_canary_cmd() {
+    # Rsync command
+    cmd="$(command -v rsync)"
+    # Rsync options
+    cmd="${cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'"
+    # Rsync local source
+    cmd="${cmd} ${CANARY_FILE}"
+    # Rsync remote destination
+    cmd="${cmd} root@${SSH_SERVER}:/var/backup/"
+
+    # output final command
+    echo "${cmd}"
+}
 sync_tasks() {
     n=0
     server=""
@@ -382,86 +475,48 @@ sync_tasks() {
     SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
     SSH_PORT=$(echo "${server}" | cut -d':' -f2)
 
+    log "START SYNC_TASKS - server=${server}"
+
+    # default paths, depending on system
     if [ "${SYSTEM}" = "linux" ]; then
-        rep="/bin /boot /lib /opt /sbin /usr"
+        default_includes="/bin /boot /lib /opt /sbin /usr"
     else
-        rep="/bsd /bin /sbin /usr"
+        default_includes="/bsd /bin /sbin /usr"
     fi
 
-    log "START SYNC_TASKS - server=${server}"
+    # reset Rsync log file
+    if [ -n "$(command -v truncate)" ]; then 
+        truncate -s 0 "${RSYNC_LOGFILE}"
+    else
+        printf "" > "${RSYNC_LOGFILE}"
+    fi
 
-    update-evobackup-canary --who "${PROGNAME}"
+    # Build the final Rsync command
+    rsync_main_cmd=$(build_rsync_main_cmd)
+
+    # … log it
+    log "SYNC_TASKS - Rsync main command : ${rsync_main_cmd}"
+
+    # … execute it
+    eval "${rsync_main_cmd}"
 
-    # /!\ DO NOT USE COMMENTS in the rsync command /!\
-    # It breaks the command and destroys data, simply remove (or add) lines.
-
-    # Remote shell command
-    RSH_COMMAND="ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'"
-
-    # ignore check because we want it to split the different arguments to $rep
-    # shellcheck disable=SC2086
-    rsync --archive                                        \
-        --itemize-changes --stats --human-readable         \
-        --relative --partial                               \
-        --delete --delete-excluded --force --ignore-errors \
-        --exclude "dev"                                    \
-        --exclude "lost+found"                             \
-        --exclude ".nfs.*"                                 \
-        --exclude "/usr/doc"                               \
-        --exclude "/usr/obj"                               \
-        --exclude "/usr/share/doc"                         \
-        --exclude "/usr/src"                               \
-        --exclude "/var/apt"                               \
-        --exclude "/var/cache"                             \
-        --exclude "/var/lib/amavis/amavisd.sock"           \
-        --exclude "/var/lib/amavis/tmp"                    \
-        --exclude "/var/lib/clamav/*.tmp"                  \
-        --exclude "/var/lib/elasticsearch"                 \
-        --exclude "/var/lib/metche"                        \
-        --exclude "/var/lib/munin/*tmp*"                   \
-        --exclude "/var/db/munin/*.tmp"                    \
-        --exclude "/var/lib/mysql"                         \
-        --exclude "/var/lib/php5"                          \
-        --exclude "/var/lib/php/sessions"                  \
-        --exclude "/var/lib/postgres"                      \
-        --exclude "/var/lib/postgresql"                    \
-        --exclude "/var/lib/sympa"                         \
-        --exclude "/var/lock"                              \
-        --exclude "/var/run"                               \
-        --exclude "/var/spool/postfix"                     \
-        --exclude "/var/spool/smtpd"                       \
-        --exclude "/var/spool/squid"                       \
-        --exclude "/var/state"                             \
-        --exclude "/var/tmp"                               \
-        --exclude "lxc/*/rootfs/tmp"                       \
-        --exclude "lxc/*/rootfs/usr/doc"                   \
-        --exclude "lxc/*/rootfs/usr/obj"                   \
-        --exclude "lxc/*/rootfs/usr/share/doc"             \
-        --exclude "lxc/*/rootfs/usr/src"                   \
-        --exclude "lxc/*/rootfs/var/apt"                   \
-        --exclude "lxc/*/rootfs/var/cache"                 \
-        --exclude "lxc/*/rootfs/var/lib/php5"              \
-        --exclude "lxc/*/rootfs/var/lib/php/sessions"      \
-        --exclude "lxc/*/rootfs/var/lock"                  \
-        --exclude "lxc/*/rootfs/var/log"                   \
-        --exclude "lxc/*/rootfs/var/run"                   \
-        --exclude "lxc/*/rootfs/var/state"                 \
-        --exclude "lxc/*/rootfs/var/tmp"                   \
-        --exclude "/home/mysqltmp"                         \
-            ${rep}                                         \
-            /etc                                           \
-            /root                                          \
-            /var                                           \
-            /home                                          \
-            /zzz_evobackup_canary                          \
-        -e "${RSH_COMMAND}"                                \
-        "root@${SSH_SERVER}:/var/backup/"                  \
-            | tail -30 >> "${LOGFILE}"
-
-    rsync_rc=$?
-    if [ ${rsync_rc} -ne 0 ]; then
-        error "rsync returned an error ${rsync_rc}, check ${LOGFILE}"
+    rsync_main_rc=$?
+
+    # Copy last lines of rsync log to the main log
+    tail -n 30 "${RSYNC_LOGFILE}" >> "${LOGFILE}"
+
+    if [ ${rsync_main_rc} -ne 0 ]; then
+        error "rsync returned an error ${rsync_main_rc}, check ${LOGFILE}"
         rc=201
+    else
+        # Build the canary Rsync command
+        rsync_canary_cmd=$(build_rsync_canary_cmd)
+
+        # … log it
+        log "SYNC_TASKS - Rsync canary command : ${rsync_canary_cmd}"
+
+        # … execute it
+        eval "${rsync_canary_cmd}"
     fi
 
     log "STOP SYNC_TASKS - server=${server}"
@@ -565,8 +620,16 @@ main() {
         fi
     fi
     echo "$$" > "${PIDFILE}"
+
+    # Initialize a list of files to delete at exit
+    # Any file added to the list will also be deleted at exit
+    temp_files="${PIDFILE}"
+
     # shellcheck disable=SC2064
-    trap "rm -f ${PIDFILE}" EXIT
+    trap "rm -f ${temp_files}" EXIT
+
+    # Update canary to keep track of each run
+    update-evobackup-canary --who "${PROGNAME}"
 
     if [ "${LOCAL_TASKS}" = "1" ]; then
         local_tasks
@@ -597,8 +660,6 @@ export LC_ALL=C
 
 # Error on unassigned variable
 set -u
-# Fail if a pipeline member returns an error (cf. https://sipb.mit.edu/doc/safe-shell/)
-set -o pipefail
 
 # Default return-code (0 == succes)
 rc=0

server/CHANGELOG.md

@@ -18,6 +18,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security
 
+## [22.11] - 2022-11-28
+
+### Added
+
+* check-canary: new subcommand to check canary files and content
+
+### Changed
+
+* stats: filter active jails and columnize the output
+
 ## [22.07] - 2022-07-20
 
 ### Changed

server/README.md

@@ -39,7 +39,7 @@ This architecture is as secure as SSH, Rsync, chroot and iptables are.
 
 ## Install
 
-See the [installation guide](docs/install.md) for instructions.
+See the [installation guide](https://intra.evolix.net/OutilsInternes/bkctld) for instructions.
 
 ## Testing
 
@@ -49,6 +49,12 @@ You can deploy test environments with Vagrant :
 vagrant up
 ~~~
 
+To destroy Vagrant VMs :
+
+~~~
+vagrant destroy
+~~~
+
 ### Deployment
 
 Run `vagrant rsync-auto` in a terminal for automatic synchronization of

server/bkctld

@@ -53,6 +53,9 @@ while :; do
         -f|--force)
             export FORCE=1
             ;;
+        --no-header)
+            export HEADER=0
+            ;;
         *)
             # Default case: If no more options then break out of the loop.
             break
@@ -64,7 +67,7 @@ done
 
 subcommand="${1:-}"
 case "${subcommand}" in
-    "inc" | "rm" | "check-jails" | "check-setup" | "stats" | "list")
+    "inc" | "rm" | "check-jails" | "check-setup" | "check-canary" | "stats" | "list")
         "${LIBDIR}/bkctld-${subcommand}"
     ;;
     "check")
@@ -116,7 +119,9 @@ case "${subcommand}" in
     ;;
     "status")
         jail_name="${2:-}"
-        printf '%-30s %-10s %-10s %-25s %-20s\n' 'JAIL NAME' 'STATUS' 'PORT' 'RETENTION (DAY/MONTH)' 'IP'
+        if [ "${HEADER}" = "1" ]; then
+            printf '%-30s %-10s %-10s %-25s %-20s\n' 'JAIL NAME' 'STATUS' 'PORT' 'RETENTION (DAY/MONTH)' 'IP'
+        fi
         if [ "${jail_name}" = "all" ] || [ -z "${jail_name}" ]; then
             for jail in $("${LIBDIR}/bkctld-list"); do
                 "${LIBDIR}/bkctld-${subcommand}" "${jail}"

server/docs/install.md

@@ -2,15 +2,7 @@
 
 ## Install from package
 
-A Debian package is available in the Evolix repository
-
-~~~
-echo "deb http://pub.evolix.net/ stretch" >> /etc/apt/sources.list
-apt update
-apt install bkctld
-~~~
-
-Then edit `/etc/default/bkctld`
+The install documentation is [here](https://intra.evolix.net/OutilsInternes/bkctld)
 
 ## Instal from sources
 

server/lib/bkctld-check-canary

@@ -0,0 +1,59 @@
+#!/bin/sh
+#
+# Description: check canary file
+# Usage: check-canary [<jailname>|all]
+#
+
+# shellcheck source=./includes
+LIBDIR="$(dirname $0)" && . "${LIBDIR}/includes"
+
+return=0
+nb_crit=0
+nb_warn=0
+nb_ok=0
+nb_unkn=0
+output=""
+
+date=$(date +"%Y-%m-%d")
+
+# Check each jail status
+
+check_jail() {
+    jail_name=$1
+
+    jail_path=$(jail_path "${jail_name}")
+    canary_absolute_file="${jail_path}/var/backup/${CANARY_RELATIVE_FILE}"
+
+    if [ -f "${canary_absolute_file}" ]; then
+        if grep --quiet --fixed-string "${date}" "${canary_absolute_file}"; then
+            nb_ok=$((nb_ok + 1))
+            output="${output}OK - ${jail_name} - entries found for ${date} in ${CANARY_RELATIVE_FILE} file\n"
+        else
+            nb_crit=$((nb_crit + 1))
+            output="${output}CRITICAL - ${jail_name} - No entry for ${date} in ${CANARY_RELATIVE_FILE} file\n"
+            [ "${return}" -le 2 ] && return=2
+        fi
+    else
+        nb_crit=$((nb_crit + 1))
+        output="${output}CRITICAL - ${jail_name} - missing ${CANARY_RELATIVE_FILE} file\n"
+        [ "${return}" -le 2 ] && return=2
+    fi
+}
+
+for jail_name in $(jails_list); do
+    check_jail "${jail_name}"
+done
+
+[ "${return}" -ge 0 ] && header="OK"
+[ "${return}" -ge 1 ] && header="WARNING"
+[ "${return}" -ge 2 ] && header="CRITICAL"
+[ "${return}" -ge 3 ] && header="UNKNOWN"
+
+printf "%s - %s UNK / %s CRIT / %s WARN / %s OK\n\n" "${header}" "${nb_unkn}" "${nb_crit}" "${nb_warn}" "${nb_ok}"
+
+printf "${output}" | grep -E "^UNKNOWN"
+printf "${output}" | grep -E "^CRITICAL"
+printf "${output}" | grep -E "^WARNING"
+printf "${output}" | grep -E "^OK"
+
+exit "${return}"

server/lib/bkctld-stats

@@ -15,27 +15,29 @@ ionice -c3 "${DUC}" index -d "${IDX_FILE}" "${JAILDIR}"
 touch "${INDEX_DIR}/.lastrun.duc"
 EOF
 
-[ ! -f "${INDEX_DIR}/.lastrun.duc" ] && notice "First run of DUC always in progress ..." && exit 0
+[ ! -f "${INDEX_DIR}/.lastrun.duc" ] && notice "First run of DUC still in progress ..." && exit 0
 [ ! -f ${IDX_FILE} ] && error "Index file doesn't exits !"
 
 printf "Last update of index file : "
 stat --format=%Y "${INDEX_DIR}/.lastrun.duc" | xargs -i -n1 date -R -d "@{}"
-echo "<jail> <size> <incs> <lastconn>" | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $1, $2, $3, $4); }'
 
 duc_output=$(mktemp)
 stat_output=$(mktemp)
 incs_output=$(mktemp)
+jail_patterns_list=$(mktemp)
 
 # shellcheck disable=SC2064
-trap "rm ${duc_output} ${incs_output} ${stat_output}" 0
+trap "rm ${duc_output} ${incs_output} ${stat_output} ${jail_patterns_list}" 0
 
-"${DUC}" ls -d "${IDX_FILE}" "${JAILDIR}" > "${duc_output}"
+"${DUC}" ls --database "${IDX_FILE}" "${JAILDIR}" > "${duc_output}"
 
-awk '{ print $2 }' "${duc_output}" | while read jail_name; do
+jails_list | sed -e "s/^\(.*\)$/\\\\b\1\\\\b/" > "${jail_patterns_list}"
+
+grep -f "${jail_patterns_list}" "${duc_output}" | awk '{ print $2 }' | while read jail_name; do
     jail_path=$(jail_path "${jail_name}")
     stat --format=%Y "${jail_path}/var/log/lastlog" | xargs -i -n1 date -d "@{}" "+%d-%m-%Y" >> "${stat_output}"
 
-    incs_policy_file=$(current_jail_incs_policy_file ${jail_name})
+    incs_policy_file=$(current_jail_incs_policy_file "${jail_name}")
     incs_policy="0"
     if [ -r "${incs_policy_file}" ]; then
         days=$(grep "^\+" "${incs_policy_file}" | grep --count "day")
@@ -45,4 +47,7 @@ awk '{ print $2 }' "${duc_output}" | while read jail_name; do
     echo "${incs_policy}" >> "${incs_output}"
 done
 
-paste "${duc_output}" "${incs_output}" "${stat_output}" | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $2, $1, $3, $4); }'
+(
+    echo "<jail> <size> <incs> <lastconn>"
+    paste "${duc_output}" "${incs_output}" "${stat_output}" | awk '{ printf("%s %s %s %s\n", $2, $1, $3, $4); }'
+) | column -t

server/lib/bkctld-status

@@ -1,7 +1,7 @@
 #!/bin/sh
 #
 # Description: Display status of SSH server
-# Usage: status [<jailname>|all]
+# Usage: [--no-header] status [<jailname>|all]
 #
 
 # shellcheck source=./includes

server/lib/includes

@@ -6,7 +6,7 @@
 
 [ -f /etc/default/bkctld ] && . /etc/default/bkctld
 
-VERSION="22.07"
+VERSION="22.11"
 
 LIBDIR=${LIBDIR:-/usr/lib/bkctld}
 CONFDIR="${CONFDIR:-/etc/evobackup}"
@@ -20,6 +20,7 @@ LOCKDIR="${LOCKDIR:-/run/lock/bkctld}"
 ARCHIVESDIR="${ARCHIVESDIR:-${BACKUP_PARTITION}/archives}"
 INDEX_DIR="${INDEX_DIR:-${BACKUP_PARTITION}/index}"
 IDX_FILE="${IDX_FILE:-${INDEX_DIR}/bkctld-jails.idx}"
+CANARY_RELATIVE_FILE="${CANARY_RELATIVE_FILE:-/zzz_evobackup_canary}"
 SSHD_PID="${SSHD_PID:-/run/sshd.pid}"
 SSHD_CONFIG="${SSHD_CONFIG:-/etc/ssh/sshd_config}"
 AUTHORIZED_KEYS="${AUTHORIZED_KEYS:-/root/.ssh/authorized_keys}"
@@ -29,6 +30,7 @@ CRITICAL="${CRITICAL:-48}"
 WARNING="${WARNING:-24}"
 DUC=$(command -v duc-nox || command -v duc)
 FORCE="${FORCE:-0}"
+HEADER="${HEADER:-1}"
 
 show_version() {
     cat <<END
@@ -62,6 +64,13 @@ EOF
     printf "\n"
 }
 
+is_quiet() {
+    test ${QUIET} -eq 1
+}
+is_verbose() {
+    test ${VERBOSE} -eq 1
+}
+
 log_date() {
     echo "[$(date +"%Y-%m-%d %H:%M:%S")]"
 }

server/test/checks.bats

@@ -252,3 +252,25 @@ OUT
     assert_failure
 }
 # TODO: write many more tests for bkctld-check-incs
+
+@test "Check-canary fails if a canary file doesn't exist" {
+    run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}"
+    assert_equal "$status" "2"
+    assert_line "CRITICAL - ${JAILNAME} - missing /zzz_evobackup_canary file"
+}
+
+@test "Check-canary fails if a canary is missing today's entries" {
+    today="$(date +%Y-%m-%d)"
+    touch "${JAILPATH}/var/backup/zzz_evobackup_canary"
+
+    run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}"
+    assert_equal "$status" "2"
+    assert_line "CRITICAL - ${JAILNAME} - No entry for ${today} in /zzz_evobackup_canary file"
+}
+
+@test "Check-canary succeeds if a canary has today's entries" {
+    echo "$(date "+%FT%T%z") bats-test" >> "${JAILPATH}/var/backup/zzz_evobackup_canary"
+
+    run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}"
+    assert_success
+}