Merge pull request 'fix-tempfiles' (#72) from fix-tempfiles into master

Reviewed-on: #72

.Jenkinsfile

@@ -1,24 +1,15 @@
 pipeline {
-    agent { label 'docker' }
+    agent { label 'sbuild' }
     stages {
         stage('Build Debian package') {
-            agent {
-                docker {
-                    image 'evolix/gbp:bullseye'
-                    args '-u root --privileged -v /tmp:/tmp'
-                }
-            }
             when {
                 branch 'debian'
             }
             steps {
                 script {
-                    sh 'mk-build-deps --install --remove debian/control'
-                    sh 'rm -rf source'
-                    sh "gbp clone --debian-branch=$GIT_BRANCH $GIT_URL source"
-                    sh 'cd source && git checkout $GIT_BRANCH && gbp buildpackage -us -uc'
+                    sh 'gbp buildpackage'
                 }
-                archiveArtifacts allowEmptyArchive: true, artifacts: '*.gz,*.bz2,*.xz,*.deb,*.dsc,*.changes,*.buildinfo,lintian.txt'
+                archiveArtifacts allowEmptyArchive: true, artifacts: 'build-area/*.gz,build-area/*.bz2,build-area/*.xz,build-area/*.deb,build-area/*.dsc,build-area/*.changes,build-area/*.buildinfo,build-area/*.build,build-area/lintian.txt'
             }
         }
 
@@ -28,10 +19,7 @@ pipeline {
             }
             steps {
                 script {
-                    sh 'echo Dummy line to remove once something actually happens.'
-                 /* No crendentials yet
-                    sh 'rsync -avP /tmp/bkctld/ droneci@pub.evolix.net:/home/droneci/bkctld/'
-                  */
+                    sh 'rsync -avP build-area/bkctld*.deb build-area/bkctld*.changes build-area/bkctld*.buildinfo pub.evolix.org:/srv/upload/'
                 }
             }
         }

.drone.yml

@@ -1,46 +0,0 @@
-kind: pipeline
-name: default
-
-steps:
-- name: fetch
-  image: alpine/git
-  commands:
-  - git fetch --tags
-
-- name: build debian package
-  image: evolix/gbp:bullseye
-  branches:
-  - debian
-  commands:
-  - mk-build-deps --install --remove debian/control
-  - git clean --force
-  - gbp buildpackage -us -uc
-  volumes:
-  - name: tmp
-    path: /tmp
-  when:
-    branch:
-    - debian
-
-- name: upload debian package
-  image: drillster/drone-rsync
-  settings:
-    hosts: ["pub.evolix.net"]
-    port: 22
-    user: droneci
-    key:
-      from_secret: drone_private_key
-    target: /home/droneci/bkctld/
-    source: /tmp/bkctld/
-    delete: true
-  volumes:
-  - name: tmp
-    path: /tmp
-  when:
-    branch:
-    - debian
-
-volumes:
-- name: tmp
-  host:
-    path: /tmp

.gitignore

@@ -0,0 +1,2 @@
+*.swp
+.vagrant

client/CHANGELOG.md

@@ -10,6 +10,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+### Deprecated
+
+### Removed
+
+### Fixed
+
+### Security
+
+## [22.12]
+### Changed
+
 * Use --dump-dir instead of --backup-dir to suppress dump-server-state warning
 * Do not use rsync compression
 * Replace rsync option --verbose by --itemize-changes
@@ -17,17 +28,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * update-evobackup-canary: do not use GNU date, for it to be compatible with OpenBSD
 * Add AGPL License and README
 * Script now depends on Bash
-
-### Deprecated
+* tolerate absence of mtr or traceroute
+* Only one loop for all Redis instances
+* remodel how we build the rsync command
+* use sub shells instead of moving around
+* Separate Rsync for the canary file if the main Rsync has finished without errors
 
 ### Removed
 
+* No more fallback if dump-server-state is missing
+
 ### Fixed
 
 * Make start_time and stop_time compatible with OpenBSD
 
-### Security
-
 ## [22.03]
 
 Split client and server parts of the project

client/README.md

@@ -1,3 +1,3 @@
-Pour l'installation de `zzz_evobackup`, voir <https://intra.evolix.net/Installation_jail_backup_Evolix#installation-du-client-evobackup>
+Pour l'installation de `zzz_evobackup`, voir <https://intra.evolix.net/OutilsInternes/EvoBackupClient#installer-et-configurer-le-client-evobackup>
 
-Pour `update-evobackup-canary`, voir <https://intra.evolix.net/OutilsInternes/update-evobackup-canary>
+Pour `update-evobackup-canary`, voir <https://intra.evolix.net/OutilsInternes/update-evobackup-canary>

client/zzz_evobackup

@@ -18,7 +18,7 @@
 
 ##### Configuration ###################################################
 
-VERSION="22.05"
+VERSION="22.12"
 
 # email adress for notifications
 MAIL=jdoe@example.com
@@ -48,6 +48,9 @@ PIDFILE="/var/run/${PROGNAME}.pid"
 # Customize the log path if you have multiple scripts and with separate logs
 LOGFILE="/var/log/evobackup.log"
 
+# Full Rsync log file, reset each time
+RSYNC_LOGFILE="/var/log/${PROGNAME}.rsync.log"
+
 HOSTNAME=$(hostname)
 
 DATE_FORMAT="%Y-%m-%d %H:%M:%S"
@@ -57,7 +60,77 @@ DATE_FORMAT="%Y-%m-%d %H:%M:%S"
 # Enable/disable sync tasks (default: enabled)
 : "${SYNC_TASKS:=1}"
 
-##### SETUP AND FUNCTIONS #############################################
+CANARY_FILE="/zzz_evobackup_canary"
+
+# Source paths can be customized
+# Empty lines, and lines containing # or ; are ignored
+# NOTE: remember to single-quote paths if they contain globs (*)
+#       and you want to defer expansion
+RSYNC_INCLUDES="
+/etc
+/root
+/var
+/home
+"
+
+# Excluded paths can be customized
+# Empty lines, and lines beginning with # or ; are ignored
+# NOTE: remember to single-quote paths if they contain globs (*)
+#       and you want to defer expansion
+RSYNC_EXCLUDES="
+/dev
+/proc
+/run
+/sys
+/tmp
+/usr/doc
+/usr/obj
+/usr/share/doc
+/usr/src
+/var/apt
+/var/cache
+'/var/db/munin/*.tmp'
+/var/lib/amavis/amavisd.sock
+/var/lib/amavis/tmp
+/var/lib/amavis/virusmails
+'/var/lib/clamav/*.tmp'
+/var/lib/elasticsearch
+/var/lib/metche
+/var/lib/mongodb
+'/var/lib/munin/*tmp*'
+/var/lib/mysql
+/var/lib/php/sessions
+/var/lib/php5
+/var/lib/postgres
+/var/lib/postgresql
+/var/lib/sympa
+/var/lock
+/var/run
+/var/spool/postfix
+/var/spool/smtpd
+/var/spool/squid
+/var/state
+/var/tmp
+lost+found
+'.nfs.*'
+'lxc/*/rootfs/tmp'
+'lxc/*/rootfs/usr/doc'
+'lxc/*/rootfs/usr/obj'
+'lxc/*/rootfs/usr/share/doc'
+'lxc/*/rootfs/usr/src'
+'lxc/*/rootfs/var/apt'
+'lxc/*/rootfs/var/cache'
+'lxc/*/rootfs/var/lib/php5'
+'lxc/*/rootfs/var/lib/php/sessions'
+'lxc/*/rootfs/var/lock'
+'lxc/*/rootfs/var/run'
+'lxc/*/rootfs/var/state'
+'lxc/*/rootfs/var/tmp'
+/home/mysqltmp
+"
+
+
+##### FUNCTIONS #######################################################
 
 local_tasks() {
     log "START LOCAL_TASKS"
@@ -76,7 +149,7 @@ local_tasks() {
     # rm -rf ${LOCAL_BACKUP_DIR}/mysql
     # rm -rf ${LOCAL_BACKUP_DIR}/mysqlhotcopy
     # rm -rf /home/mysqldump
-    # rm -f ${LOCAL_BACKUP_DIR}/*.err ${LOCAL_BACKUP_DIR}/**/*.err
+    # find ${LOCAL_BACKUP_DIR}/ -type f -name '*.err' -delete
 
     ## example with global and compressed mysqldump
     # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
@@ -90,7 +163,7 @@ local_tasks() {
     ## example with compressed SQL dump (with data) for each databases
     # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
     # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
-    # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
+    # | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)"); do
     #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i 2> ${LOCAL_BACKUP_DIR}/${i}.err | gzip --best > ${LOCAL_BACKUP_DIR}/mysql/${i}.sql.gz
     #     last_rc=$?
     #     if [ ${last_rc} -ne 0 ]; then
@@ -119,7 +192,7 @@ local_tasks() {
     ## example with SQL dump (schema only, no data) for each databases
     # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
     # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
-    # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
+    # | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)"); do
     #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases $i 2> ${LOCAL_BACKUP_DIR}/${i}.schema.err > ${LOCAL_BACKUP_DIR}/mysql/${i}.schema.sql
     #     last_rc=$?
     #     if [ ${last_rc} -ne 0 ]; then
@@ -141,7 +214,7 @@ local_tasks() {
 
     ## example with two dumps for each table (.sql/.txt) for all databases
     # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
-    # | egrep -v "^(Database|information_schema|performance_schema|sys)" ); do
+    # | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)" ); do
     #     mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump
     #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \
     #  --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i 2> /home/mysqldump/$i.err"
@@ -163,7 +236,7 @@ local_tasks() {
 
     ## example for multiples MySQL instances
     # mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3)
-    # grep -E "^port\s*=\s*\d*" /etc/mysql/my.cnf |while read instance; do
+    # grep --extended-regexp "^port\s*=\s*\d*" /etc/mysql/my.cnf | while read instance; do
     #     instance=$(echo "$instance"|awk '{ print $3 }')
     #     if [ "$instance" != "3306" ]
     #     then
@@ -182,13 +255,16 @@ local_tasks() {
     # rm -rf ${LOCAL_BACKUP_DIR}/pg.*.gz
     # rm -rf ${LOCAL_BACKUP_DIR}/pg-backup.tar
     # rm -rf ${LOCAL_BACKUP_DIR}/postgresql/*
+
     ## example with pg_dumpall (warning: you need space in ~postgres)
     # su - postgres -c "pg_dumpall > ~/pg.dump.bak"
     # mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/
+
     ## another method with gzip directly piped
-    # cd /var/lib/postgresql
-    # sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
-    # cd - > /dev/null
+    # (
+    #    cd /var/lib/postgresql;
+    #    sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
+    # )
 
     ## example with all tables from MYBASE excepts TABLE1 and TABLE2
     # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE
@@ -199,9 +275,11 @@ local_tasks() {
     ## example with compressed PostgreSQL dump for each databases
     # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/postgresql
     # chown postgres:postgres ${LOCAL_BACKUP_DIR}/postgresql
-    # dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*)
-    # 
-    # for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done
+    # (
+    #    cd /var/lib/postgresql
+    #    dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*)
+    #    for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done
+    # )
 
     ## MongoDB
 
@@ -221,15 +299,14 @@ local_tasks() {
     ## Purge previous dumps
     # rm -rf ${LOCAL_BACKUP_DIR}/redis/
     # rm -rf ${LOCAL_BACKUP_DIR}/redis-*
-    ## example with copy .rdb file
-    ## for the default instance :
-    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/redis/
-    # cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/redis/
-    ## for multiple instances :
-    # for instance in $(ls -d /var/lib/redis-*); do
-    #     name=$(basename $instance)
-    #     mkdir -p ${LOCAL_BACKUP_DIR}/${name}
-    #     cp -a ${instance}/dump.rdb ${LOCAL_BACKUP_DIR}/${name}
+    ## Copy dump.rdb file for each found instance
+    # for instance in $(find /var/lib/ -mindepth 1 -maxdepth 1 '(' -type d -o -type l ')' -name 'redis*'); do
+    #     if [ -f "${instance}/dump.rdb" ]; then
+    #         name=$(basename $instance)
+    #         mkdir -p ${LOCAL_BACKUP_DIR}/${name}
+    #         cp -a "${instance}/dump.rdb" "${LOCAL_BACKUP_DIR}/${name}"
+    #         gzip "${LOCAL_BACKUP_DIR}/${name}/dump.rdb"
+    #     fi
     # done
 
     ## ElasticSearch
@@ -248,7 +325,7 @@ local_tasks() {
     # else
     #     echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
     # fi
-    ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
+    ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
     # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
     #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
     # done
@@ -258,68 +335,35 @@ local_tasks() {
     ## RabbitMQ
 
     ## export config
-    #rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> "${LOGFILE}"
+    # rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> "${LOGFILE}"
 
     ## MegaCli config
 
-    #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null
+    # megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null
 
     ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls)
-    for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do
-        mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr}
-        traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1
-    done
+    network_targets="8.8.8.8 www.evolix.fr travaux.evolix.net"
+    mtr_bin=$(command -v mtr)
+    if [ -n "${mtr_bin}" ]; then
+        for addr in ${network_targets}; do
+            ${mtr_bin} -r "${addr}" > "${LOCAL_BACKUP_DIR}/mtr-${addr}"
+        done
+    fi
+    traceroute_bin=$(command -v traceroute)
+    if [ -n "${traceroute_bin}" ]; then
+        for addr in ${network_targets}; do
+            ${traceroute_bin} -n "${addr}" > "${LOCAL_BACKUP_DIR}/traceroute-${addr}" 2>&1
+        done
+    fi
 
     server_state_dir="${LOCAL_BACKUP_DIR}/server-state"
 
     dump_server_state_bin=$(command -v dump-server-state)
-
-    if [ "${SYSTEM}" = "linux" ]; then
-        if [ -n "${dump_server_state_bin}" ]; then
-            ${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
-            last_rc=$?
-            if [ ${last_rc} -ne 0 ]; then
-                error "dump-server-state returned an error ${last_rc}, check ${server_state_dir}"
-                rc=1
-            fi
-        else
-            mkdir -p "${server_state_dir}"
-
-            ## Dump system and kernel versions
-            uname -a > ${server_state_dir}/uname.txt
-
-            ## Dump process with ps
-            ps auwwx > ${server_state_dir}/ps.txt
-
-            ## Dump network connections with ss
-            ss -taupen > ${server_state_dir}/netstat.txt
-
-            ## List Debian packages
-            dpkg -l > ${server_state_dir}/packages
-            dpkg --get-selections > ${server_state_dir}/packages.getselections
-            apt-cache dumpavail > ${server_state_dir}/packages.available
-
-            ## Dump iptables
-            if [ -x /sbin/iptables ]; then
-                { /sbin/iptables -L -n -v; /sbin/iptables -t filter -L -n -v; } > ${server_state_dir}/iptables.txt
-            fi
-
-            ## Dump findmnt(8) output
-            FINDMNT_BIN=$(command -v findmnt)
-            if [ -x "${FINDMNT_BIN}" ]; then
-                ${FINDMNT_BIN} > ${server_state_dir}/findmnt.txt
-            fi
-
-            ## Dump MBR / table partitions
-            disks=$(lsblk -l | grep disk | grep -v -E '(drbd|fd[0-9]+)' | awk '{print $1}')
-            for disk in ${disks}; do
-                dd if="/dev/${disk}" of="${server_state_dir}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)"
-                fdisk -l "/dev/${disk}" > "${server_state_dir}/partitions-${disk}" 2>&1
-            done
-            cat ${server_state_dir}/partitions-* > ${server_state_dir}/partitions
-        fi
+    if [ -z "${dump_server_state_bin}" ]; then
+        error "dump-server-state is missing"
+        rc=1
     else
-        if [ -n "${dump_server_state_bin}" ]; then
+        if [ "${SYSTEM}" = "linux" ]; then
             ${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
             last_rc=$?
             if [ ${last_rc} -ne 0 ]; then
@@ -327,37 +371,93 @@ local_tasks() {
                 rc=1
             fi
         else
-            mkdir -p "${server_state_dir}"
-
-            ## Dump system and kernel versions
-            uname -a > ${server_state_dir}/uname
-
-            ## Dump process with ps
-            ps auwwx > ${server_state_dir}/ps.out
-
-            ## Dump network connections with fstat
-            fstat | head -1 > ${server_state_dir}/netstat.out
-            fstat | grep internet >> ${server_state_dir}/netstat.out
-
-            ## List OpenBSD packages
-            pkg_info -m > ${server_state_dir}/packages
-
-            ## Dump MBR / table partitions
-            disklabel sd0 > ${server_state_dir}/partitions
-
-            ## Dump pf infos
-            pfctl -sa > ${server_state_dir}/pfctl-sa.txt
+            ${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
+            last_rc=$?
+            if [ ${last_rc} -ne 0 ]; then
+                error "dump-server-state returned an error ${last_rc}, check ${server_state_dir}"
+                rc=1
+            fi
         fi
     fi
 
     ## Dump rights
-    #getfacl -R /var  > ${server_state_dir}/rights-var.txt
-    #getfacl -R /etc  > ${server_state_dir}/rights-etc.txt
-    #getfacl -R /usr  > ${server_state_dir}/rights-usr.txt
-    #getfacl -R /home > ${server_state_dir}/rights-home.txt
+    # getfacl -R /var  > ${server_state_dir}/rights-var.txt
+    # getfacl -R /etc  > ${server_state_dir}/rights-etc.txt
+    # getfacl -R /usr  > ${server_state_dir}/rights-usr.txt
+    # getfacl -R /home > ${server_state_dir}/rights-home.txt
 
     log "STOP LOCAL_TASKS"
 }
+build_rsync_main_cmd() {
+    ###################################################################
+    # /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ #
+    ###################################################################
+    # DO NOT USE COMMENTS in rsync lines                              #
+    # DO NOT ADD WHITESPACES AFTER \ in rsync lines                   #
+    # It breaks the command and destroys data                         #
+    # You should not modify this, unless you are really REALLY sure   #
+    ###################################################################
+
+    # Create a temp file for excludes and includes
+    includes_file="$(mktemp --tmpdir "${PROGNAME}.includes.XXXXXX")"
+    excludes_file="$(mktemp --tmpdir "${PROGNAME}.excludes.XXXXXX")"
+    # … and add them to the list of files to delete at exit
+    temp_files="${includes_file} ${excludes_file}"
+    trap "rm -f ${temp_files}" EXIT
+
+    # Store includes/excludes in files
+    # without blank lines of comments (# or ;)
+    echo "${RSYNC_INCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${includes_file}"
+    echo "${RSYNC_EXCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${excludes_file}"
+
+    # Rsync command
+    cmd="$(command -v rsync)"
+
+    # Rsync main options
+    cmd="${cmd} --archive"
+    cmd="${cmd} --itemize-changes"
+    cmd="${cmd} --quiet"
+    cmd="${cmd} --stats"
+    cmd="${cmd} --human-readable"
+    cmd="${cmd} --relative"
+    cmd="${cmd} --partial"
+    cmd="${cmd} --delete"
+    cmd="${cmd} --delete-excluded"
+    cmd="${cmd} --force"
+    cmd="${cmd} --ignore-errors"
+    cmd="${cmd} --log-file=${RSYNC_LOGFILE}"
+    cmd="${cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'"
+
+    # Rsync excludes
+    while read line ; do
+        cmd="${cmd} --exclude ${line}"
+    done < "${excludes_file}"
+
+    # Rsync local sources
+    cmd="${cmd} ${default_includes}"
+    while read line ; do
+        cmd="${cmd} ${line}"
+    done < "${includes_file}"
+
+    # Rsync remote destination
+    cmd="${cmd} root@${SSH_SERVER}:/var/backup/"
+
+    # output final command
+    echo "${cmd}"
+}
+build_rsync_canary_cmd() {
+    # Rsync command
+    cmd="$(command -v rsync)"
+    # Rsync options
+    cmd="${cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'"
+    # Rsync local source
+    cmd="${cmd} ${CANARY_FILE}"
+    # Rsync remote destination
+    cmd="${cmd} root@${SSH_SERVER}:/var/backup/"
+
+    # output final command
+    echo "${cmd}"
+}
 sync_tasks() {
     n=0
     server=""
@@ -382,86 +482,48 @@ sync_tasks() {
     SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
     SSH_PORT=$(echo "${server}" | cut -d':' -f2)
 
-    if [ "${SYSTEM}" = "linux" ]; then
-        rep="/bin /boot /lib /opt /sbin /usr"
-    else
-        rep="/bsd /bin /sbin /usr"
-    fi
-
     log "START SYNC_TASKS - server=${server}"
 
-    update-evobackup-canary --who "${PROGNAME}"
+    # default paths, depending on system
+    if [ "${SYSTEM}" = "linux" ]; then
+        default_includes="/bin /boot /lib /opt /sbin /usr"
+    else
+        default_includes="/bsd /bin /sbin /usr"
+    fi
 
-    # /!\ DO NOT USE COMMENTS in the rsync command /!\
-    # It breaks the command and destroys data, simply remove (or add) lines.
+    # reset Rsync log file
+    if [ -n "$(command -v truncate)" ]; then 
+        truncate -s 0 "${RSYNC_LOGFILE}"
+    else
+        printf "" > "${RSYNC_LOGFILE}"
+    fi
 
-    # Remote shell command
-    RSH_COMMAND="ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'"
+    # Build the final Rsync command
+    rsync_main_cmd=$(build_rsync_main_cmd)
 
-    # ignore check because we want it to split the different arguments to $rep
-    # shellcheck disable=SC2086
-    rsync --archive                                        \
-        --itemize-changes --stats --human-readable         \
-        --relative --partial                               \
-        --delete --delete-excluded --force --ignore-errors \
-        --exclude "dev"                                    \
-        --exclude "lost+found"                             \
-        --exclude ".nfs.*"                                 \
-        --exclude "/usr/doc"                               \
-        --exclude "/usr/obj"                               \
-        --exclude "/usr/share/doc"                         \
-        --exclude "/usr/src"                               \
-        --exclude "/var/apt"                               \
-        --exclude "/var/cache"                             \
-        --exclude "/var/lib/amavis/amavisd.sock"           \
-        --exclude "/var/lib/amavis/tmp"                    \
-        --exclude "/var/lib/clamav/*.tmp"                  \
-        --exclude "/var/lib/elasticsearch"                 \
-        --exclude "/var/lib/metche"                        \
-        --exclude "/var/lib/munin/*tmp*"                   \
-        --exclude "/var/db/munin/*.tmp"                    \
-        --exclude "/var/lib/mysql"                         \
-        --exclude "/var/lib/php5"                          \
-        --exclude "/var/lib/php/sessions"                  \
-        --exclude "/var/lib/postgres"                      \
-        --exclude "/var/lib/postgresql"                    \
-        --exclude "/var/lib/sympa"                         \
-        --exclude "/var/lock"                              \
-        --exclude "/var/run"                               \
-        --exclude "/var/spool/postfix"                     \
-        --exclude "/var/spool/smtpd"                       \
-        --exclude "/var/spool/squid"                       \
-        --exclude "/var/state"                             \
-        --exclude "/var/tmp"                               \
-        --exclude "lxc/*/rootfs/tmp"                       \
-        --exclude "lxc/*/rootfs/usr/doc"                   \
-        --exclude "lxc/*/rootfs/usr/obj"                   \
-        --exclude "lxc/*/rootfs/usr/share/doc"             \
-        --exclude "lxc/*/rootfs/usr/src"                   \
-        --exclude "lxc/*/rootfs/var/apt"                   \
-        --exclude "lxc/*/rootfs/var/cache"                 \
-        --exclude "lxc/*/rootfs/var/lib/php5"              \
-        --exclude "lxc/*/rootfs/var/lib/php/sessions"      \
-        --exclude "lxc/*/rootfs/var/lock"                  \
-        --exclude "lxc/*/rootfs/var/log"                   \
-        --exclude "lxc/*/rootfs/var/run"                   \
-        --exclude "lxc/*/rootfs/var/state"                 \
-        --exclude "lxc/*/rootfs/var/tmp"                   \
-        --exclude "/home/mysqltmp"                         \
-            ${rep}                                         \
-            /etc                                           \
-            /root                                          \
-            /var                                           \
-            /home                                          \
-            /zzz_evobackup_canary                          \
-        -e "${RSH_COMMAND}"                                \
-        "root@${SSH_SERVER}:/var/backup/"                  \
-            | tail -30 >> "${LOGFILE}"
+    # … log it
+    log "SYNC_TASKS - Rsync main command : ${rsync_main_cmd}"
 
-    rsync_rc=$?
-    if [ ${rsync_rc} -ne 0 ]; then
-        error "rsync returned an error ${rsync_rc}, check ${LOGFILE}"
+    # … execute it
+    eval "${rsync_main_cmd}"
+
+    rsync_main_rc=$?
+
+    # Copy last lines of rsync log to the main log
+    tail -n 30 "${RSYNC_LOGFILE}" >> "${LOGFILE}"
+
+    if [ ${rsync_main_rc} -ne 0 ]; then
+        error "rsync returned an error ${rsync_main_rc}, check ${LOGFILE}"
         rc=201
+    else
+        # Build the canary Rsync command
+        rsync_canary_cmd=$(build_rsync_canary_cmd)
+
+        # … log it
+        log "SYNC_TASKS - Rsync canary command : ${rsync_canary_cmd}"
+
+        # … execute it
+        eval "${rsync_canary_cmd}"
     fi
 
     log "STOP SYNC_TASKS - server=${server}"
@@ -565,8 +627,16 @@ main() {
         fi
     fi
     echo "$$" > "${PIDFILE}"
+
+    # Initialize a list of files to delete at exit
+    # Any file added to the list will also be deleted at exit
+    temp_files="${PIDFILE}"
+
     # shellcheck disable=SC2064
-    trap "rm -f ${PIDFILE}" EXIT
+    trap "rm -f ${temp_files}" EXIT
+
+    # Update canary to keep track of each run
+    update-evobackup-canary --who "${PROGNAME}"
 
     if [ "${LOCAL_TASKS}" = "1" ]; then
         local_tasks
@@ -597,8 +667,6 @@ export LC_ALL=C
 
 # Error on unassigned variable
 set -u
-# Fail if a pipeline member returns an error (cf. https://sipb.mit.edu/doc/safe-shell/)
-set -o pipefail
 
 # Default return-code (0 == succes)
 rc=0

server/CHANGELOG.md

@@ -18,6 +18,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security
 
+## [22.11] - 2022-11-28
+
+### Added
+
+* check-canary: new subcommand to check canary files and content
+
+### Changed
+
+* stats: filter active jails and columnize the output
+
+## [22.07] - 2022-07-20
+
+### Changed
+
+* check-setup: check minifirewall version only if minifirewall is present
+* check-setup: get minifirewall version from internal variable (there is no other backward compatible way)
+* check-setup: use findmnt with mountpoint instead of target
+
 ## [22.06] - 2022-06-28
 
 ### Added

server/README.md

@@ -4,7 +4,7 @@ Bkctld (aka server-side evobackup)
 bkctld helps you manage the receiving side of a backup infrastructure.
 It is licensed under the AGPLv3.
 
-With bkctld you create and manage "jails". They contain a chrooted and dedicated SSH server, with it's own TCP port and optionnaly it's own set of iptables rules.
+With bkctld you create and manage "jails". They contain a chrooted and dedicated SSH server, with its own TCP port and optionally its own set of iptables rules.
 
 With bkctld you can have hundreds of jails, one for each client to push its data (using Rsync/SFTP). Each client can only see its own data.
 
@@ -30,9 +30,7 @@ This volume can also be encrypted with **LUKS**.
 
 ## Security considerations
 
-The client obviously has access to its uploaded data (in the chroot), but the timestamped copies are outside the chroot, to reduce the risk or complete backup erasure from a compromised client.
-
-Since the client connects to the backup server with root, it can mess with the jail and destroy the data. But the timestamped copies are out of reach because outside of the chroot.
+The client obviously has access to its uploaded data (in the chroot), but the timestamped copies are outside the chroot, to reduce the risk of complete backup erasure from a compromised client.
 
 It means that **if the client server is compromised**, an attacker can destroy the latest copy of the backed up data, but not the timestamped copies.
 And **if the backup server is compromised** an attacker has complete access to all the backup data (inside and outside the jails), but they don't have any access to the client.
@@ -41,7 +39,7 @@ This architecture is as secure as SSH, Rsync, chroot and iptables are.
 
 ## Install
 
-See the [installation guide](docs/install.md) for instructions.
+See the [installation guide](https://intra.evolix.net/OutilsInternes/bkctld) for instructions.
 
 ## Testing
 
@@ -51,6 +49,12 @@ You can deploy test environments with Vagrant :
 vagrant up
 ~~~
 
+To destroy Vagrant VMs :
+
+~~~
+vagrant destroy
+~~~
+
 ### Deployment
 
 Run `vagrant rsync-auto` in a terminal for automatic synchronization of
@@ -77,6 +81,8 @@ vagrant@buster-btrfs $ sudo -i
 root@buster-btrfs # bats /vagrant/test/*.bats
 ~~~
 
+[comment]: <> (* pour vim)
+
 You should shellcheck your bats files, but with shellcheck > 0.4.6, because the 0.4.0 version doesn't support bats syntax.
 
 ## Usage
@@ -99,7 +105,7 @@ pandoc -f markdown \
 #### Client configuration
 
 You can backup various systems in the evobackup jails : Linux, BSD,
-Windows, macOS. The only need Rsync or an SFTP client.
+Windows, macOS. The only need is Rsync or an SFTP client.
 
 ~~~
 rsync -av -e "ssh -p SSH_PORT" /home/ root@SERVER_NAME:/var/backup/home/

server/bkctld

@@ -53,6 +53,9 @@ while :; do
         -f|--force)
             export FORCE=1
             ;;
+        --no-header)
+            export HEADER=0
+            ;;
         *)
             # Default case: If no more options then break out of the loop.
             break
@@ -64,7 +67,7 @@ done
 
 subcommand="${1:-}"
 case "${subcommand}" in
-    "inc" | "rm" | "check-jails" | "check-setup" | "stats" | "list")
+    "inc" | "rm" | "check-jails" | "check-setup" | "check-canary" | "stats" | "list")
         "${LIBDIR}/bkctld-${subcommand}"
     ;;
     "check")
@@ -116,7 +119,9 @@ case "${subcommand}" in
     ;;
     "status")
         jail_name="${2:-}"
-        printf '%-30s %-10s %-10s %-25s %-20s\n' 'JAIL NAME' 'STATUS' 'PORT' 'RETENTION (DAY/MONTH)' 'IP'
+        if [ "${HEADER}" = "1" ]; then
+            printf '%-30s %-10s %-10s %-25s %-20s\n' 'JAIL NAME' 'STATUS' 'PORT' 'RETENTION (DAY/MONTH)' 'IP'
+        fi
         if [ "${jail_name}" = "all" ] || [ -z "${jail_name}" ]; then
             for jail in $("${LIBDIR}/bkctld-list"); do
                 "${LIBDIR}/bkctld-${subcommand}" "${jail}"

server/bkctld.conf

@@ -15,3 +15,5 @@
 #LOGLEVEL=6
 #NODE=''
 #ARCHIVESDIR='/backup/archives'
+#WARNING=48
+#CRITICAL=72

server/docs/install.md

@@ -2,15 +2,7 @@
 
 ## Install from package
 
-A Debian package is available in the Evolix repository
-
-~~~
-echo "deb http://pub.evolix.net/ stretch" >> /etc/apt/sources.list
-apt update
-apt install bkctld
-~~~
-
-Then edit `/etc/default/bkctld`
+The install documentation is [here](https://intra.evolix.net/OutilsInternes/bkctld)
 
 ## Instal from sources
 
@@ -19,17 +11,17 @@ Warning: `cp`-ing the files without `-n` or `-i` will replace existing files !
 ~~~
 # git clone https://gitea.evolix.org/evolix/evobackup.git
 # cd evobackup
-# cp bkctld /usr/local/sbin/
+# cp server/bkctld /usr/local/sbin/
 # mkdir -p /usr/local/lib/bkctld
-# cp lib/* /usr/local/lib/bkctld/
+# cp server/lib/* /usr/local/lib/bkctld/
 # mkdir -p /usr/local/share/bkctld
-# cp tpl/* /usr/local/share/bkctld/
-# cp bkctld.service /lib/systemd/system/
+# cp server/tpl/* /usr/local/share/bkctld/
+# cp server/bkctld.service /lib/systemd/system/
 # mkdir -p /usr/local/share/doc/bkctld
-# cp zzz_evobackup /usr/local/share/doc/bkctld/
+# cp client/zzz_evobackup /usr/local/share/doc/bkctld/
 # mkdir -p /usr/local/share/bash_completion/
-# cp bash_completion /usr/local/share/bash_completion/bkctld
-# cp bkctld.conf /etc/default/bkctld
+# cp server/bash_completion /usr/local/share/bash_completion/bkctld
+# cp server/bkctld.conf /etc/default/bkctld
 ~~~
 
 ## Chroot dependencies

server/lib/bkctld-check-canary

@@ -0,0 +1,59 @@
+#!/bin/sh
+#
+# Description: check canary file
+# Usage: check-canary [<jailname>|all]
+#
+
+# shellcheck source=./includes
+LIBDIR="$(dirname $0)" && . "${LIBDIR}/includes"
+
+return=0
+nb_crit=0
+nb_warn=0
+nb_ok=0
+nb_unkn=0
+output=""
+
+date=$(date +"%Y-%m-%d")
+
+# Check each jail status
+
+check_jail() {
+    jail_name=$1
+
+    jail_path=$(jail_path "${jail_name}")
+    canary_absolute_file="${jail_path}/var/backup/${CANARY_RELATIVE_FILE}"
+
+    if [ -f "${canary_absolute_file}" ]; then
+        if grep --quiet --fixed-string "${date}" "${canary_absolute_file}"; then
+            nb_ok=$((nb_ok + 1))
+            output="${output}OK - ${jail_name} - entries found for ${date} in ${CANARY_RELATIVE_FILE} file\n"
+        else
+            nb_crit=$((nb_crit + 1))
+            output="${output}CRITICAL - ${jail_name} - No entry for ${date} in ${CANARY_RELATIVE_FILE} file\n"
+            [ "${return}" -le 2 ] && return=2
+        fi
+    else
+        nb_crit=$((nb_crit + 1))
+        output="${output}CRITICAL - ${jail_name} - missing ${CANARY_RELATIVE_FILE} file\n"
+        [ "${return}" -le 2 ] && return=2
+    fi
+}
+
+for jail_name in $(jails_list); do
+    check_jail "${jail_name}"
+done
+
+[ "${return}" -ge 0 ] && header="OK"
+[ "${return}" -ge 1 ] && header="WARNING"
+[ "${return}" -ge 2 ] && header="CRITICAL"
+[ "${return}" -ge 3 ] && header="UNKNOWN"
+
+printf "%s - %s UNK / %s CRIT / %s WARN / %s OK\n\n" "${header}" "${nb_unkn}" "${nb_crit}" "${nb_warn}" "${nb_ok}"
+
+printf "${output}" | grep -E "^UNKNOWN"
+printf "${output}" | grep -E "^CRITICAL"
+printf "${output}" | grep -E "^WARNING"
+printf "${output}" | grep -E "^OK"
+
+exit "${return}"

server/lib/bkctld-check-setup

@@ -16,7 +16,7 @@ output=""
 
 # Verify backup partition is mounted and writable
 
-findmnt -O rw --target "${BACKUP_PARTITION}" > /dev/null
+findmnt -O rw --mountpoint "${BACKUP_PARTITION}" > /dev/null
 if [ "$?" -ne 0 ]; then
     nb_crit=$((nb_crit + 1))
     output="${output}CRITICAL - Backup disk \`/backup' is not mounted (or read-only) !\n"
@@ -29,11 +29,12 @@ fi
 # Check if the firewall file is sourced
 
 minifirewall_config=/etc/default/minifirewall
-minifirewall_version=$(/etc/init.d/minifirewall status | head -1 | cut -d ' ' -f 3)
 
 if [ -n "${FIREWALL_RULES}" ] \
 && [ -r "${FIREWALL_RULES}" ] \
 && [ -f "${minifirewall_config}" ]; then
+    minifirewall_version=$(grep -E -o "^VERSION=(\S+)" /etc/init.d/minifirewall | head -1 | cut -d '=' -f 2 | tr -d "'" | tr -d '"')
+
     if [ -n "${minifirewall_version}" ] && dpkg --compare-versions "${minifirewall_version}" ge "22.03"; then
         # Minifirewall 22.03+ includes files automatically
         nb_ok=$((nb_ok + 1))

server/lib/bkctld-rename

@@ -51,15 +51,15 @@ if dry_run; then
 else
     mv "${jail_path}" "${new_jail_path}"
 fi
-if dry_run; then
-    if [ -d "${incs_path}" ]; then
+
+if [ -d "${incs_path}" ]; then
+    if dry_run; then
         echo "[dry-run] rename ${incs_path} to ${new_incs_path}"
-    fi
-else
-    if [ -d "${incs_path}" ]; then
+    else
         mv "${incs_path}" "${new_incs_path}"
     fi
 fi
+
 if [ -d "${jail_config_dir}" ]; then
     if dry_run; then
         echo "[dry-run] rename ${jail_config_dir} to ${new_jail_config_dir}"

server/lib/bkctld-stats

@@ -15,27 +15,29 @@ ionice -c3 "${DUC}" index -d "${IDX_FILE}" "${JAILDIR}"
 touch "${INDEX_DIR}/.lastrun.duc"
 EOF
 
-[ ! -f "${INDEX_DIR}/.lastrun.duc" ] && notice "First run of DUC always in progress ..." && exit 0
+[ ! -f "${INDEX_DIR}/.lastrun.duc" ] && notice "First run of DUC still in progress ..." && exit 0
 [ ! -f ${IDX_FILE} ] && error "Index file doesn't exits !"
 
 printf "Last update of index file : "
 stat --format=%Y "${INDEX_DIR}/.lastrun.duc" | xargs -i -n1 date -R -d "@{}"
-echo "<jail> <size> <incs> <lastconn>" | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $1, $2, $3, $4); }'
 
 duc_output=$(mktemp)
 stat_output=$(mktemp)
 incs_output=$(mktemp)
+jail_patterns_list=$(mktemp)
 
 # shellcheck disable=SC2064
-trap "rm ${duc_output} ${incs_output} ${stat_output}" 0
+trap "rm ${duc_output} ${incs_output} ${stat_output} ${jail_patterns_list}" 0
 
-"${DUC}" ls -d "${IDX_FILE}" "${JAILDIR}" > "${duc_output}"
+"${DUC}" ls --database "${IDX_FILE}" "${JAILDIR}" > "${duc_output}"
 
-awk '{ print $2 }' "${duc_output}" | while read jail_name; do
+jails_list | sed -e "s/^\(.*\)$/\\\\b\1\\\\b/" > "${jail_patterns_list}"
+
+grep -f "${jail_patterns_list}" "${duc_output}" | awk '{ print $2 }' | while read jail_name; do
     jail_path=$(jail_path "${jail_name}")
     stat --format=%Y "${jail_path}/var/log/lastlog" | xargs -i -n1 date -d "@{}" "+%d-%m-%Y" >> "${stat_output}"
 
-    incs_policy_file=$(current_jail_incs_policy_file ${jail_name})
+    incs_policy_file=$(current_jail_incs_policy_file "${jail_name}")
     incs_policy="0"
     if [ -r "${incs_policy_file}" ]; then
         days=$(grep "^\+" "${incs_policy_file}" | grep --count "day")
@@ -45,4 +47,7 @@ awk '{ print $2 }' "${duc_output}" | while read jail_name; do
     echo "${incs_policy}" >> "${incs_output}"
 done
 
-paste "${duc_output}" "${incs_output}" "${stat_output}" | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $2, $1, $3, $4); }'
+(
+    echo "<jail> <size> <incs> <lastconn>"
+    paste "${duc_output}" "${incs_output}" "${stat_output}" | awk '{ printf("%s %s %s %s\n", $2, $1, $3, $4); }'
+) | column -t

server/lib/bkctld-status

@@ -1,7 +1,7 @@
 #!/bin/sh
 #
 # Description: Display status of SSH server
-# Usage: status [<jailname>|all]
+# Usage: [--no-header] status [<jailname>|all]
 #
 
 # shellcheck source=./includes

server/lib/includes

@@ -6,7 +6,7 @@
 
 [ -f /etc/default/bkctld ] && . /etc/default/bkctld
 
-VERSION="22.04"
+VERSION="22.11"
 
 LIBDIR=${LIBDIR:-/usr/lib/bkctld}
 CONFDIR="${CONFDIR:-/etc/evobackup}"
@@ -20,6 +20,7 @@ LOCKDIR="${LOCKDIR:-/run/lock/bkctld}"
 ARCHIVESDIR="${ARCHIVESDIR:-${BACKUP_PARTITION}/archives}"
 INDEX_DIR="${INDEX_DIR:-${BACKUP_PARTITION}/index}"
 IDX_FILE="${IDX_FILE:-${INDEX_DIR}/bkctld-jails.idx}"
+CANARY_RELATIVE_FILE="${CANARY_RELATIVE_FILE:-/zzz_evobackup_canary}"
 SSHD_PID="${SSHD_PID:-/run/sshd.pid}"
 SSHD_CONFIG="${SSHD_CONFIG:-/etc/ssh/sshd_config}"
 AUTHORIZED_KEYS="${AUTHORIZED_KEYS:-/root/.ssh/authorized_keys}"
@@ -29,6 +30,7 @@ CRITICAL="${CRITICAL:-48}"
 WARNING="${WARNING:-24}"
 DUC=$(command -v duc-nox || command -v duc)
 FORCE="${FORCE:-0}"
+HEADER="${HEADER:-1}"
 
 show_version() {
     cat <<END
@@ -62,6 +64,13 @@ EOF
     printf "\n"
 }
 
+is_quiet() {
+    test ${QUIET} -eq 1
+}
+is_verbose() {
+    test ${VERBOSE} -eq 1
+}
+
 log_date() {
     echo "[$(date +"%Y-%m-%d %H:%M:%S")]"
 }
@@ -127,7 +136,7 @@ is_btrfs() {
 
     inode=$(stat --format=%i "${path}")
 
-    test $inode -eq 256
+    test "$inode" -eq 256
 }
 
 # Returns the list of jails found in the "jails" directory (default)

server/test/checks.bats

@@ -252,3 +252,25 @@ OUT
     assert_failure
 }
 # TODO: write many more tests for bkctld-check-incs
+
+@test "Check-canary fails if a canary file doesn't exist" {
+    run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}"
+    assert_equal "$status" "2"
+    assert_line "CRITICAL - ${JAILNAME} - missing /zzz_evobackup_canary file"
+}
+
+@test "Check-canary fails if a canary is missing today's entries" {
+    today="$(date +%Y-%m-%d)"
+    touch "${JAILPATH}/var/backup/zzz_evobackup_canary"
+
+    run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}"
+    assert_equal "$status" "2"
+    assert_line "CRITICAL - ${JAILNAME} - No entry for ${today} in /zzz_evobackup_canary file"
+}
+
+@test "Check-canary succeeds if a canary has today's entries" {
+    echo "$(date "+%FT%T%z") bats-test" >> "${JAILPATH}/var/backup/zzz_evobackup_canary"
+
+    run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}"
+    assert_success
+}