From d2731230ceb39e071c147183407f053b49e6cb4b Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 28 Oct 2022 14:08:03 +0200 Subject: [PATCH 1/6] remodel how we build the rsync command * use a log file for rsync * build the command argument by argument, without backslashes * move excludes into a file --- client/CHANGELOG.md | 1 + client/zzz_evobackup | 196 +++++++++++++++++++++++++++---------------- 2 files changed, 126 insertions(+), 71 deletions(-) diff --git a/client/CHANGELOG.md b/client/CHANGELOG.md index ad915f4..9b4d387 100644 --- a/client/CHANGELOG.md +++ b/client/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Script now depends on Bash * tolerate absence of mtr or traceroute * Only one loop for all Redis instances +* remodel how we build the rsync command ### Deprecated diff --git a/client/zzz_evobackup b/client/zzz_evobackup index 43566d0..28c6eff 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -48,6 +48,9 @@ PIDFILE="/var/run/${PROGNAME}.pid" # Customize the log path if you have multiple scripts and with separate logs LOGFILE="/var/log/evobackup.log" +# Full Rsync log file, reset each time +RSYNC_LOGFILE="/var/log/${PROGNAME}.rsync.log" + HOSTNAME=$(hostname) DATE_FORMAT="%Y-%m-%d %H:%M:%S" @@ -392,18 +395,68 @@ sync_tasks() { SSH_SERVER=$(echo "${server}" | cut -d':' -f1) SSH_PORT=$(echo "${server}" | cut -d':' -f2) - if [ "${SYSTEM}" = "linux" ]; then - rep="/bin /boot /lib /opt /sbin /usr" - else - rep="/bsd /bin /sbin /usr" - fi - log "START SYNC_TASKS - server=${server}" - update-evobackup-canary --who "${PROGNAME}" + # default paths, depending on system + if [ "${SYSTEM}" = "linux" ]; then + default_includes="/bin /boot /lib /opt /sbin /usr" + else + default_includes="/bsd /bin /sbin /usr" + fi - # Remote shell command - RSH_COMMAND="ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'" + # Create a temp file for excludes + excludes_file="$(mktemp --suffix=.excludes "${PROGNAME}.XXXXXX")" + # … and add it to the list of files to delete at exit + temp_files="${temp_files} ${excludes_file}" + + # Excluded paths can be customized + cat >> "${excludes_file}" < Only remove (or add) lines. # + # You should not modify this, unless you are really REALLY sure # ################################################################### - # ignore check because we want it to split the different arguments to $rep - # shellcheck disable=SC2086 - rsync --archive \ - --itemize-changes --stats --human-readable \ - --relative --partial \ - --delete --delete-excluded --force --ignore-errors \ - --exclude "dev" \ - --exclude "lost+found" \ - --exclude ".nfs.*" \ - --exclude "/usr/doc" \ - --exclude "/usr/obj" \ - --exclude "/usr/share/doc" \ - --exclude "/usr/src" \ - --exclude "/var/apt" \ - --exclude "/var/cache" \ - --exclude "/var/lib/amavis/amavisd.sock" \ - --exclude "/var/lib/amavis/tmp" \ - --exclude "/var/lib/clamav/*.tmp" \ - --exclude "/var/lib/elasticsearch" \ - --exclude "/var/lib/metche" \ - --exclude "/var/lib/munin/*tmp*" \ - --exclude "/var/db/munin/*.tmp" \ - --exclude "/var/lib/mongodb" \ - --exclude "/var/lib/mysql" \ - --exclude "/var/lib/php5" \ - --exclude "/var/lib/php/sessions" \ - --exclude "/var/lib/postgres" \ - --exclude "/var/lib/postgresql" \ - --exclude "/var/lib/sympa" \ - --exclude "/var/lock" \ - --exclude "/var/run" \ - --exclude "/var/spool/postfix" \ - --exclude "/var/spool/smtpd" \ - --exclude "/var/spool/squid" \ - --exclude "/var/state" \ - --exclude "/var/tmp" \ - --exclude "lxc/*/rootfs/tmp" \ - --exclude "lxc/*/rootfs/usr/doc" \ - --exclude "lxc/*/rootfs/usr/obj" \ - --exclude "lxc/*/rootfs/usr/share/doc" \ - --exclude "lxc/*/rootfs/usr/src" \ - --exclude "lxc/*/rootfs/var/apt" \ - --exclude "lxc/*/rootfs/var/cache" \ - --exclude "lxc/*/rootfs/var/lib/php5" \ - --exclude "lxc/*/rootfs/var/lib/php/sessions" \ - --exclude "lxc/*/rootfs/var/lock" \ - --exclude "lxc/*/rootfs/var/log" \ - --exclude "lxc/*/rootfs/var/run" \ - --exclude "lxc/*/rootfs/var/state" \ - --exclude "lxc/*/rootfs/var/tmp" \ - --exclude "/home/mysqltmp" \ - ${rep} \ - /etc \ - /root \ - /var \ - /home \ - /zzz_evobackup_canary \ - -e "${RSH_COMMAND}" \ - "root@${SSH_SERVER}:/var/backup/" \ - | tail -30 >> "${LOGFILE}" + # Rsync command + rsync_cmd="$(command -v rsync)" + + # Rsync main options + rsync_cmd="${rsync_cmd} --archive" + rsync_cmd="${rsync_cmd} --itemize-changes" + rsync_cmd="${rsync_cmd} --quiet" + rsync_cmd="${rsync_cmd} --stats" + rsync_cmd="${rsync_cmd} --human-readable" + rsync_cmd="${rsync_cmd} --relative" + rsync_cmd="${rsync_cmd} --partial" + rsync_cmd="${rsync_cmd} --delete" + rsync_cmd="${rsync_cmd} --delete-excluded" + rsync_cmd="${rsync_cmd} --force" + rsync_cmd="${rsync_cmd} --ignore-errors" + rsync_cmd="${rsync_cmd} --exclude-from=${excludes_file}" + rsync_cmd="${rsync_cmd} --log-file=${RSYNC_LOGFILE}" + rsync_cmd="${rsync_cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'" + + # Rsync local sources + rsync_cmd="${rsync_cmd} ${default_includes}" # Default includes are platform specific + rsync_cmd="${rsync_cmd} /etc" + rsync_cmd="${rsync_cmd} /root" + rsync_cmd="${rsync_cmd} /var" + rsync_cmd="${rsync_cmd} /home" # Consider removing /home for a system-only backup + rsync_cmd="${rsync_cmd} /zzz_evobackup_canary" # keep the canary file! + + # Rsync remote destination + rsync_cmd="${rsync_cmd} root@${SSH_SERVER}:/var/backup/" + + # log excludes, on one line, to keep a reference + excludes_log="SYNC_TASKS - Rsync excludes :" + while read line; do + excludes_log="${excludes_log} ${line}" + done < "${excludes_file}" + log "${excludes_log}" + + log "SYNC_TASKS - Rsync command : ${rsync_cmd}" + + # reset Rsync log file + if command -v truncate; then + truncate -s 0 "${RSYNC_LOGFILE}" + else + printf "" > "${RSYNC_LOGFILE}" + fi + + # execute Rsync command + eval "${rsync_cmd}" rsync_rc=$? if [ ${rsync_rc} -ne 0 ]; then @@ -481,6 +522,11 @@ sync_tasks() { rc=201 fi + ################################################################### + + # Copy last lines of rsync log to the main log + tail -n 30 "${RSYNC_LOGFILE}" >> "${LOGFILE}" + log "STOP SYNC_TASKS - server=${server}" } @@ -582,8 +628,16 @@ main() { fi fi echo "$$" > "${PIDFILE}" + + # Initialize a list of files to delete at exit + # Any file added to the list will also be deleted at exit + temp_files="${PIDFILE}" + # shellcheck disable=SC2064 - trap "rm -f ${PIDFILE}" EXIT + trap "rm -f ${temp_files}" EXIT + + # Update canary to keep track of each run + update-evobackup-canary --who "${PROGNAME}" if [ "${LOCAL_TASKS}" = "1" ]; then local_tasks -- 2.39.2 From ab17b4db3d4bb656033d24e72feaf8cdddb12681 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 28 Oct 2022 16:36:24 +0200 Subject: [PATCH 2/6] prefer inline excludes, but still use a temp file to prepare --- client/zzz_evobackup | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/client/zzz_evobackup b/client/zzz_evobackup index 28c6eff..6cca866 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -405,7 +405,7 @@ sync_tasks() { fi # Create a temp file for excludes - excludes_file="$(mktemp --suffix=.excludes "${PROGNAME}.XXXXXX")" + excludes_file="$(mktemp "${PROGNAME}.excludes.XXXXXX")" # … and add it to the list of files to delete at exit temp_files="${temp_files} ${excludes_file}" @@ -482,10 +482,18 @@ END_OF_EXCLUDES rsync_cmd="${rsync_cmd} --delete-excluded" rsync_cmd="${rsync_cmd} --force" rsync_cmd="${rsync_cmd} --ignore-errors" - rsync_cmd="${rsync_cmd} --exclude-from=${excludes_file}" rsync_cmd="${rsync_cmd} --log-file=${RSYNC_LOGFILE}" rsync_cmd="${rsync_cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'" + # Rsync excludes + while read line ; do + # Ignore blank lines, and lines beginning with # or ; + exclude=$(echo "${line}" | grep --extended-regexp "^[^;#]+") + if [ -n "${exclude}" ]; then + rsync_cmd="${rsync_cmd} --exclude ${exclude}" + fi + done < "${excludes_file}" + # Rsync local sources rsync_cmd="${rsync_cmd} ${default_includes}" # Default includes are platform specific rsync_cmd="${rsync_cmd} /etc" @@ -497,17 +505,10 @@ END_OF_EXCLUDES # Rsync remote destination rsync_cmd="${rsync_cmd} root@${SSH_SERVER}:/var/backup/" - # log excludes, on one line, to keep a reference - excludes_log="SYNC_TASKS - Rsync excludes :" - while read line; do - excludes_log="${excludes_log} ${line}" - done < "${excludes_file}" - log "${excludes_log}" - log "SYNC_TASKS - Rsync command : ${rsync_cmd}" # reset Rsync log file - if command -v truncate; then + if [ -n "$(command -v truncate)" ]; then truncate -s 0 "${RSYNC_LOGFILE}" else printf "" > "${RSYNC_LOGFILE}" -- 2.39.2 From 76a7275d1bd37576c33645d4681f607591c93048 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 28 Oct 2022 16:53:09 +0200 Subject: [PATCH 3/6] extract function to build rsync command --- client/zzz_evobackup | 123 +++++++++++++++++++++++++------------------ 1 file changed, 71 insertions(+), 52 deletions(-) diff --git a/client/zzz_evobackup b/client/zzz_evobackup index 6cca866..fe49ca0 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -371,6 +371,58 @@ local_tasks() { log "STOP LOCAL_TASKS" } +build_rsync_cmd() { + ################################################################### + # /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ # + ################################################################### + # DO NOT USE COMMENTS in rsync lines # + # DO NOT ADD WHITESPACES AFTER \ in rsync lines # + # It breaks the command and destroys data # + # You should not modify this, unless you are really REALLY sure # + ################################################################### + + # Rsync command + cmd="$(command -v rsync)" + + # Rsync main options + cmd="${cmd} --archive" + cmd="${cmd} --itemize-changes" + cmd="${cmd} --quiet" + cmd="${cmd} --stats" + cmd="${cmd} --human-readable" + cmd="${cmd} --relative" + cmd="${cmd} --partial" + cmd="${cmd} --delete" + cmd="${cmd} --delete-excluded" + cmd="${cmd} --force" + cmd="${cmd} --ignore-errors" + cmd="${cmd} --log-file=${RSYNC_LOGFILE}" + cmd="${cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'" + + # Rsync excludes + while read line ; do + # Ignore lines containing # or ; (anywhere) + exclude=$(echo "${line}" | grep --invert-match --extended-regexp "[;#]") + if [ -n "${exclude}" ]; then + cmd="${cmd} --exclude ${exclude}" + fi + done < "${excludes_file}" + + # Rsync local sources + cmd="${cmd} ${default_includes}" # Default includes are platform specific + while read line ; do + # Ignore blank lines, and lines beginning with # or ; + include=$(echo "${line}" | grep --extended-regexp "^[^;#]+") + if [ -n "${include}" ]; then + cmd="${cmd} ${include}" + fi + done < "${includes_file}" + + # Rsync remote destination + cmd="${cmd} root@${SSH_SERVER}:/var/backup/" + + echo "${cmd}" +} sync_tasks() { n=0 server="" @@ -404,10 +456,11 @@ sync_tasks() { default_includes="/bsd /bin /sbin /usr" fi - # Create a temp file for excludes + # Create a temp file for excludes and includes excludes_file="$(mktemp "${PROGNAME}.excludes.XXXXXX")" - # … and add it to the list of files to delete at exit - temp_files="${temp_files} ${excludes_file}" + includes_file="$(mktemp "${PROGNAME}.includes.XXXXXX")" + # … and add them to the list of files to delete at exit + temp_files="${temp_files} ${excludes_file} ${includes_file}" # Excluded paths can be customized cat >> "${excludes_file}" <> "${includes_file}" < "${RSYNC_LOGFILE}" fi - # execute Rsync command + # Build the final Rsync command + rsync_cmd=$(build_rsync_cmd) + + # … log it + log "SYNC_TASKS - Rsync command : ${rsync_cmd}" + + # … execute it eval "${rsync_cmd}" rsync_rc=$? -- 2.39.2 From 8ab072ed5fd5c5f0bf45a48028a13b3f1653d97f Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 28 Oct 2022 18:29:09 +0200 Subject: [PATCH 4/6] includes/excludes are pushed in the config section --- client/zzz_evobackup | 155 +++++++++++++++++++++---------------------- 1 file changed, 77 insertions(+), 78 deletions(-) diff --git a/client/zzz_evobackup b/client/zzz_evobackup index fe49ca0..0f304bc 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -60,7 +60,68 @@ DATE_FORMAT="%Y-%m-%d %H:%M:%S" # Enable/disable sync tasks (default: enabled) : "${SYNC_TASKS:=1}" -##### SETUP AND FUNCTIONS ############################################# +# Source paths can be customized +# Empty lines, and lines containing # or ; are ignored +RSYNC_INCLUDES=" +/etc +/root +/var +/home +/zzz_evobackup_canary +" + +# Excluded paths can be customized +# Empty lines, and lines beginning with # or ; are ignored +RSYNC_EXCLUDES=" +dev +lost+found +.nfs.* +/usr/doc +/usr/obj +/usr/share/doc +/usr/src +/var/apt +/var/cache +/var/lib/amavis/amavisd.sock +/var/lib/amavis/tmp +/var/lib/clamav/*.tmp +/var/lib/elasticsearch +/var/lib/metche +/var/lib/munin/*tmp* +/var/db/munin/*.tmp +/var/lib/mongodb +/var/lib/mysql +/var/lib/php5 +/var/lib/php/sessions +/var/lib/postgres +/var/lib/postgresql +/var/lib/sympa +/var/lock +/var/run +/var/spool/postfix +/var/spool/smtpd +/var/spool/squid +/var/state +/var/tmp +lxc/*/rootfs/tmp +lxc/*/rootfs/usr/doc +lxc/*/rootfs/usr/obj +lxc/*/rootfs/usr/share/doc +lxc/*/rootfs/usr/src +lxc/*/rootfs/var/apt +lxc/*/rootfs/var/cache +lxc/*/rootfs/var/lib/php5 +lxc/*/rootfs/var/lib/php/sessions +lxc/*/rootfs/var/lock +lxc/*/rootfs/var/log +lxc/*/rootfs/var/run +lxc/*/rootfs/var/state +lxc/*/rootfs/var/tmp +/home/mysqltmp +" + + +##### FUNCTIONS ####################################################### local_tasks() { log "START LOCAL_TASKS" @@ -381,6 +442,17 @@ build_rsync_cmd() { # You should not modify this, unless you are really REALLY sure # ################################################################### + # Create a temp file for excludes and includes + includes_file="$(mktemp "${PROGNAME}.includes.XXXXXX")" + excludes_file="$(mktemp "${PROGNAME}.excludes.XXXXXX")" + # … and add them to the list of files to delete at exit + temp_files="${temp_files} ${includes_file} ${excludes_file}" + + # Store includes/excludes in files + # without blank lines of comments (# or ;) + echo "${RSYNC_INCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${includes_file}" + echo "${RSYNC_EXCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${excludes_file}" + # Rsync command cmd="$(command -v rsync)" @@ -401,26 +473,19 @@ build_rsync_cmd() { # Rsync excludes while read line ; do - # Ignore lines containing # or ; (anywhere) - exclude=$(echo "${line}" | grep --invert-match --extended-regexp "[;#]") - if [ -n "${exclude}" ]; then - cmd="${cmd} --exclude ${exclude}" - fi + cmd="${cmd} --exclude ${line}" done < "${excludes_file}" # Rsync local sources - cmd="${cmd} ${default_includes}" # Default includes are platform specific + cmd="${cmd} ${default_includes}" while read line ; do - # Ignore blank lines, and lines beginning with # or ; - include=$(echo "${line}" | grep --extended-regexp "^[^;#]+") - if [ -n "${include}" ]; then - cmd="${cmd} ${include}" - fi + cmd="${cmd} ${line}" done < "${includes_file}" # Rsync remote destination cmd="${cmd} root@${SSH_SERVER}:/var/backup/" + # output final command echo "${cmd}" } sync_tasks() { @@ -456,70 +521,6 @@ sync_tasks() { default_includes="/bsd /bin /sbin /usr" fi - # Create a temp file for excludes and includes - excludes_file="$(mktemp "${PROGNAME}.excludes.XXXXXX")" - includes_file="$(mktemp "${PROGNAME}.includes.XXXXXX")" - # … and add them to the list of files to delete at exit - temp_files="${temp_files} ${excludes_file} ${includes_file}" - - # Excluded paths can be customized - cat >> "${excludes_file}" <> "${includes_file}" <> "${LOGFILE}" -- 2.39.2 From 232582871f28b42b311d0440f11a087bf9315882 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 28 Oct 2022 18:40:09 +0200 Subject: [PATCH 5/6] add some excludes --- client/zzz_evobackup | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/client/zzz_evobackup b/client/zzz_evobackup index 0f304bc..a9ddd86 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -73,26 +73,28 @@ RSYNC_INCLUDES=" # Excluded paths can be customized # Empty lines, and lines beginning with # or ; are ignored RSYNC_EXCLUDES=" -dev -lost+found -.nfs.* +/dev +/proc +/run +/sys +/tmp /usr/doc /usr/obj /usr/share/doc /usr/src /var/apt /var/cache +/var/db/munin/*.tmp /var/lib/amavis/amavisd.sock /var/lib/amavis/tmp /var/lib/clamav/*.tmp /var/lib/elasticsearch /var/lib/metche -/var/lib/munin/*tmp* -/var/db/munin/*.tmp /var/lib/mongodb +/var/lib/munin/*tmp* /var/lib/mysql -/var/lib/php5 /var/lib/php/sessions +/var/lib/php5 /var/lib/postgres /var/lib/postgresql /var/lib/sympa @@ -103,6 +105,8 @@ lost+found /var/spool/squid /var/state /var/tmp +lost+found +.nfs.* lxc/*/rootfs/tmp lxc/*/rootfs/usr/doc lxc/*/rootfs/usr/obj @@ -113,7 +117,6 @@ lxc/*/rootfs/var/cache lxc/*/rootfs/var/lib/php5 lxc/*/rootfs/var/lib/php/sessions lxc/*/rootfs/var/lock -lxc/*/rootfs/var/log lxc/*/rootfs/var/run lxc/*/rootfs/var/state lxc/*/rootfs/var/tmp -- 2.39.2 From 1797757459a7c17092c6afe9674f904cb2f0e772 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 28 Oct 2022 18:42:28 +0200 Subject: [PATCH 6/6] ascii whitespace --- client/zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/zzz_evobackup b/client/zzz_evobackup index a9ddd86..9871b44 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -316,7 +316,7 @@ local_tasks() { # else # echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.' # fi - ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by : + ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by : # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}' # done -- 2.39.2