evolix/evobackup
22
0
Fork 1
Code Issues 28 Pull requests 5 Releases 21 Activity

remodel how we build the rsync command #63

Merged
jlecour merged 6 commits from excludes-file into master 2022-12-27 10:43:40 +01:00
Conversation 0 Commits 6 Files changed 2 +126 -71
2 changed files with 126 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit d2731230ce - Show all commits

1
client/CHANGELOG.md
View file

@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Script now depends on Bash * Script now depends on Bash
* tolerate absence of mtr or traceroute * tolerate absence of mtr or traceroute
* Only one loop for all Redis instances * Only one loop for all Redis instances
* remodel how we build the rsync command
### Deprecated ### Deprecated

196
client/zzz_evobackup
View file

@ -48,6 +48,9 @@ PIDFILE="/var/run/${PROGNAME}.pid"
# Customize the log path if you have multiple scripts and with separate logs # Customize the log path if you have multiple scripts and with separate logs
LOGFILE="/var/log/evobackup.log" LOGFILE="/var/log/evobackup.log"
# Full Rsync log file, reset each time
RSYNC_LOGFILE="/var/log/${PROGNAME}.rsync.log"
HOSTNAME=$(hostname) HOSTNAME=$(hostname)
DATE_FORMAT="%Y-%m-%d %H:%M:%S" DATE_FORMAT="%Y-%m-%d %H:%M:%S"
@ -392,18 +395,68 @@ sync_tasks() {
SSH_SERVER=$(echo "${server}" | cut -d':' -f1) SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
SSH_PORT=$(echo "${server}" | cut -d':' -f2) SSH_PORT=$(echo "${server}" | cut -d':' -f2)
if [ "${SYSTEM}" = "linux" ]; then
rep="/bin /boot /lib /opt /sbin /usr"
else
rep="/bsd /bin /sbin /usr"
fi
log "START SYNC_TASKS - server=${server}" log "START SYNC_TASKS - server=${server}"
update-evobackup-canary --who "${PROGNAME}" # default paths, depending on system
if [ "${SYSTEM}" = "linux" ]; then
default_includes="/bin /boot /lib /opt /sbin /usr"
else
default_includes="/bsd /bin /sbin /usr"
fi
# Remote shell command # Create a temp file for excludes
RSH_COMMAND="ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'" excludes_file="$(mktemp --suffix=.excludes "${PROGNAME}.XXXXXX")"
# … and add it to the list of files to delete at exit
temp_files="${temp_files} ${excludes_file}"
# Excluded paths can be customized
cat >> "${excludes_file}" <<END_OF_EXCLUDES
dev
lost+found
.nfs.*
/usr/doc
/usr/obj
/usr/share/doc
/usr/src
/var/apt
/var/cache
/var/lib/amavis/amavisd.sock
/var/lib/amavis/tmp
/var/lib/clamav/*.tmp
/var/lib/elasticsearch
/var/lib/metche
/var/lib/munin/*tmp*
/var/db/munin/*.tmp
/var/lib/mongodb
/var/lib/mysql
/var/lib/php5
/var/lib/php/sessions
/var/lib/postgres
/var/lib/postgresql
/var/lib/sympa
/var/lock
/var/run
/var/spool/postfix
/var/spool/smtpd
/var/spool/squid
/var/state
/var/tmp
lxc/*/rootfs/tmp
lxc/*/rootfs/usr/doc
lxc/*/rootfs/usr/obj
lxc/*/rootfs/usr/share/doc
lxc/*/rootfs/usr/src
lxc/*/rootfs/var/apt
lxc/*/rootfs/var/cache
lxc/*/rootfs/var/lib/php5
lxc/*/rootfs/var/lib/php/sessions
lxc/*/rootfs/var/lock
lxc/*/rootfs/var/log
lxc/*/rootfs/var/run
lxc/*/rootfs/var/state
lxc/*/rootfs/var/tmp
/home/mysqltmp
END_OF_EXCLUDES
################################################################### ###################################################################
# /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ # # /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ #
@ -411,69 +464,57 @@ sync_tasks() {
# DO NOT USE COMMENTS in rsync lines # # DO NOT USE COMMENTS in rsync lines #
# DO NOT ADD WHITESPACES AFTER \ in rsync lines # # DO NOT ADD WHITESPACES AFTER \ in rsync lines #
# It breaks the command and destroys data # # It breaks the command and destroys data #
# => Only remove (or add) lines. # # You should not modify this, unless you are really REALLY sure #
################################################################### ###################################################################
# ignore check because we want it to split the different arguments to $rep # Rsync command
# shellcheck disable=SC2086 rsync_cmd="$(command -v rsync)"
rsync --archive \
--itemize-changes --stats --human-readable \ # Rsync main options
--relative --partial \ rsync_cmd="${rsync_cmd} --archive"
--delete --delete-excluded --force --ignore-errors \ rsync_cmd="${rsync_cmd} --itemize-changes"
--exclude "dev" \ rsync_cmd="${rsync_cmd} --quiet"
--exclude "lost+found" \ rsync_cmd="${rsync_cmd} --stats"
--exclude ".nfs.*" \ rsync_cmd="${rsync_cmd} --human-readable"
--exclude "/usr/doc" \ rsync_cmd="${rsync_cmd} --relative"
--exclude "/usr/obj" \ rsync_cmd="${rsync_cmd} --partial"
--exclude "/usr/share/doc" \ rsync_cmd="${rsync_cmd} --delete"
--exclude "/usr/src" \ rsync_cmd="${rsync_cmd} --delete-excluded"
--exclude "/var/apt" \ rsync_cmd="${rsync_cmd} --force"
--exclude "/var/cache" \ rsync_cmd="${rsync_cmd} --ignore-errors"
--exclude "/var/lib/amavis/amavisd.sock" \ rsync_cmd="${rsync_cmd} --exclude-from=${excludes_file}"
--exclude "/var/lib/amavis/tmp" \ rsync_cmd="${rsync_cmd} --log-file=${RSYNC_LOGFILE}"
--exclude "/var/lib/clamav/*.tmp" \ rsync_cmd="${rsync_cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'"
--exclude "/var/lib/elasticsearch" \
--exclude "/var/lib/metche" \ # Rsync local sources
--exclude "/var/lib/munin/*tmp*" \ rsync_cmd="${rsync_cmd} ${default_includes}" # Default includes are platform specific
--exclude "/var/db/munin/*.tmp" \ rsync_cmd="${rsync_cmd} /etc"
--exclude "/var/lib/mongodb" \ rsync_cmd="${rsync_cmd} /root"
--exclude "/var/lib/mysql" \ rsync_cmd="${rsync_cmd} /var"
--exclude "/var/lib/php5" \ rsync_cmd="${rsync_cmd} /home" # Consider removing /home for a system-only backup
--exclude "/var/lib/php/sessions" \ rsync_cmd="${rsync_cmd} /zzz_evobackup_canary" # keep the canary file!
--exclude "/var/lib/postgres" \
--exclude "/var/lib/postgresql" \ # Rsync remote destination
--exclude "/var/lib/sympa" \ rsync_cmd="${rsync_cmd} root@${SSH_SERVER}:/var/backup/"
--exclude "/var/lock" \
--exclude "/var/run" \ # log excludes, on one line, to keep a reference
--exclude "/var/spool/postfix" \ excludes_log="SYNC_TASKS - Rsync excludes :"
--exclude "/var/spool/smtpd" \ while read line; do
--exclude "/var/spool/squid" \ excludes_log="${excludes_log} ${line}"
--exclude "/var/state" \ done < "${excludes_file}"
--exclude "/var/tmp" \ log "${excludes_log}"
--exclude "lxc/*/rootfs/tmp" \
--exclude "lxc/*/rootfs/usr/doc" \ log "SYNC_TASKS - Rsync command : ${rsync_cmd}"
--exclude "lxc/*/rootfs/usr/obj" \
--exclude "lxc/*/rootfs/usr/share/doc" \ # reset Rsync log file
--exclude "lxc/*/rootfs/usr/src" \ if command -v truncate; then
--exclude "lxc/*/rootfs/var/apt" \ truncate -s 0 "${RSYNC_LOGFILE}"
--exclude "lxc/*/rootfs/var/cache" \ else
--exclude "lxc/*/rootfs/var/lib/php5" \ printf "" > "${RSYNC_LOGFILE}"
--exclude "lxc/*/rootfs/var/lib/php/sessions" \ fi
--exclude "lxc/*/rootfs/var/lock" \
--exclude "lxc/*/rootfs/var/log" \ # execute Rsync command
--exclude "lxc/*/rootfs/var/run" \ eval "${rsync_cmd}"
--exclude "lxc/*/rootfs/var/state" \
--exclude "lxc/*/rootfs/var/tmp" \
--exclude "/home/mysqltmp" \
${rep} \
/etc \
/root \
/var \
/home \
/zzz_evobackup_canary \
-e "${RSH_COMMAND}" \
"root@${SSH_SERVER}:/var/backup/" \
| tail -30 >> "${LOGFILE}"
rsync_rc=$? rsync_rc=$?
if [ ${rsync_rc} -ne 0 ]; then if [ ${rsync_rc} -ne 0 ]; then
@ -481,6 +522,11 @@ sync_tasks() {
rc=201 rc=201
fi fi
###################################################################
# Copy last lines of rsync log to the main log
tail -n 30 "${RSYNC_LOGFILE}" >> "${LOGFILE}"
log "STOP SYNC_TASKS - server=${server}" log "STOP SYNC_TASKS - server=${server}"
} }
@ -582,8 +628,16 @@ main() {
fi fi
fi fi
echo "$$" > "${PIDFILE}" echo "$$" > "${PIDFILE}"
# Initialize a list of files to delete at exit
# Any file added to the list will also be deleted at exit
temp_files="${PIDFILE}"
# shellcheck disable=SC2064 # shellcheck disable=SC2064
trap "rm -f ${PIDFILE}" EXIT trap "rm -f ${temp_files}" EXIT
# Update canary to keep track of each run
update-evobackup-canary --who "${PROGNAME}"
if [ "${LOCAL_TASKS}" = "1" ]; then if [ "${LOCAL_TASKS}" = "1" ]; then
local_tasks local_tasks