remodel how we build the rsync command (#63)
All checks were successful
gitea/evobackup/pipeline/head This commit looks good
All checks were successful
gitea/evobackup/pipeline/head This commit looks good
* use a log file for rsync * build the command argument by argument, without backslashes * move excludes into a file Co-authored-by: Jeremy Lecour <jlecour@evolix.fr> Reviewed-on: #63
This commit is contained in:
parent
140a498e28
commit
ed0645c9d2
|
@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||||
* Script now depends on Bash
|
* Script now depends on Bash
|
||||||
* tolerate absence of mtr or traceroute
|
* tolerate absence of mtr or traceroute
|
||||||
* Only one loop for all Redis instances
|
* Only one loop for all Redis instances
|
||||||
|
* remodel how we build the rsync command
|
||||||
|
|
||||||
### Deprecated
|
### Deprecated
|
||||||
|
|
||||||
|
|
|
@ -48,6 +48,9 @@ PIDFILE="/var/run/${PROGNAME}.pid"
|
||||||
# Customize the log path if you have multiple scripts and with separate logs
|
# Customize the log path if you have multiple scripts and with separate logs
|
||||||
LOGFILE="/var/log/evobackup.log"
|
LOGFILE="/var/log/evobackup.log"
|
||||||
|
|
||||||
|
# Full Rsync log file, reset each time
|
||||||
|
RSYNC_LOGFILE="/var/log/${PROGNAME}.rsync.log"
|
||||||
|
|
||||||
HOSTNAME=$(hostname)
|
HOSTNAME=$(hostname)
|
||||||
|
|
||||||
DATE_FORMAT="%Y-%m-%d %H:%M:%S"
|
DATE_FORMAT="%Y-%m-%d %H:%M:%S"
|
||||||
|
@ -57,7 +60,71 @@ DATE_FORMAT="%Y-%m-%d %H:%M:%S"
|
||||||
# Enable/disable sync tasks (default: enabled)
|
# Enable/disable sync tasks (default: enabled)
|
||||||
: "${SYNC_TASKS:=1}"
|
: "${SYNC_TASKS:=1}"
|
||||||
|
|
||||||
##### SETUP AND FUNCTIONS #############################################
|
# Source paths can be customized
|
||||||
|
# Empty lines, and lines containing # or ; are ignored
|
||||||
|
RSYNC_INCLUDES="
|
||||||
|
/etc
|
||||||
|
/root
|
||||||
|
/var
|
||||||
|
/home
|
||||||
|
/zzz_evobackup_canary
|
||||||
|
"
|
||||||
|
|
||||||
|
# Excluded paths can be customized
|
||||||
|
# Empty lines, and lines beginning with # or ; are ignored
|
||||||
|
RSYNC_EXCLUDES="
|
||||||
|
/dev
|
||||||
|
/proc
|
||||||
|
/run
|
||||||
|
/sys
|
||||||
|
/tmp
|
||||||
|
/usr/doc
|
||||||
|
/usr/obj
|
||||||
|
/usr/share/doc
|
||||||
|
/usr/src
|
||||||
|
/var/apt
|
||||||
|
/var/cache
|
||||||
|
/var/db/munin/*.tmp
|
||||||
|
/var/lib/amavis/amavisd.sock
|
||||||
|
/var/lib/amavis/tmp
|
||||||
|
/var/lib/clamav/*.tmp
|
||||||
|
/var/lib/elasticsearch
|
||||||
|
/var/lib/metche
|
||||||
|
/var/lib/mongodb
|
||||||
|
/var/lib/munin/*tmp*
|
||||||
|
/var/lib/mysql
|
||||||
|
/var/lib/php/sessions
|
||||||
|
/var/lib/php5
|
||||||
|
/var/lib/postgres
|
||||||
|
/var/lib/postgresql
|
||||||
|
/var/lib/sympa
|
||||||
|
/var/lock
|
||||||
|
/var/run
|
||||||
|
/var/spool/postfix
|
||||||
|
/var/spool/smtpd
|
||||||
|
/var/spool/squid
|
||||||
|
/var/state
|
||||||
|
/var/tmp
|
||||||
|
lost+found
|
||||||
|
.nfs.*
|
||||||
|
lxc/*/rootfs/tmp
|
||||||
|
lxc/*/rootfs/usr/doc
|
||||||
|
lxc/*/rootfs/usr/obj
|
||||||
|
lxc/*/rootfs/usr/share/doc
|
||||||
|
lxc/*/rootfs/usr/src
|
||||||
|
lxc/*/rootfs/var/apt
|
||||||
|
lxc/*/rootfs/var/cache
|
||||||
|
lxc/*/rootfs/var/lib/php5
|
||||||
|
lxc/*/rootfs/var/lib/php/sessions
|
||||||
|
lxc/*/rootfs/var/lock
|
||||||
|
lxc/*/rootfs/var/run
|
||||||
|
lxc/*/rootfs/var/state
|
||||||
|
lxc/*/rootfs/var/tmp
|
||||||
|
/home/mysqltmp
|
||||||
|
"
|
||||||
|
|
||||||
|
|
||||||
|
##### FUNCTIONS #######################################################
|
||||||
|
|
||||||
local_tasks() {
|
local_tasks() {
|
||||||
log "START LOCAL_TASKS"
|
log "START LOCAL_TASKS"
|
||||||
|
@ -249,7 +316,7 @@ local_tasks() {
|
||||||
# else
|
# else
|
||||||
# echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
|
# echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
|
||||||
# fi
|
# fi
|
||||||
## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
|
## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
|
||||||
# for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
|
# for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
|
||||||
# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
|
# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
|
||||||
# done
|
# done
|
||||||
|
@ -368,6 +435,62 @@ local_tasks() {
|
||||||
|
|
||||||
log "STOP LOCAL_TASKS"
|
log "STOP LOCAL_TASKS"
|
||||||
}
|
}
|
||||||
|
build_rsync_cmd() {
|
||||||
|
###################################################################
|
||||||
|
# /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ #
|
||||||
|
###################################################################
|
||||||
|
# DO NOT USE COMMENTS in rsync lines #
|
||||||
|
# DO NOT ADD WHITESPACES AFTER \ in rsync lines #
|
||||||
|
# It breaks the command and destroys data #
|
||||||
|
# You should not modify this, unless you are really REALLY sure #
|
||||||
|
###################################################################
|
||||||
|
|
||||||
|
# Create a temp file for excludes and includes
|
||||||
|
includes_file="$(mktemp "${PROGNAME}.includes.XXXXXX")"
|
||||||
|
excludes_file="$(mktemp "${PROGNAME}.excludes.XXXXXX")"
|
||||||
|
# … and add them to the list of files to delete at exit
|
||||||
|
temp_files="${temp_files} ${includes_file} ${excludes_file}"
|
||||||
|
|
||||||
|
# Store includes/excludes in files
|
||||||
|
# without blank lines of comments (# or ;)
|
||||||
|
echo "${RSYNC_INCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${includes_file}"
|
||||||
|
echo "${RSYNC_EXCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${excludes_file}"
|
||||||
|
|
||||||
|
# Rsync command
|
||||||
|
cmd="$(command -v rsync)"
|
||||||
|
|
||||||
|
# Rsync main options
|
||||||
|
cmd="${cmd} --archive"
|
||||||
|
cmd="${cmd} --itemize-changes"
|
||||||
|
cmd="${cmd} --quiet"
|
||||||
|
cmd="${cmd} --stats"
|
||||||
|
cmd="${cmd} --human-readable"
|
||||||
|
cmd="${cmd} --relative"
|
||||||
|
cmd="${cmd} --partial"
|
||||||
|
cmd="${cmd} --delete"
|
||||||
|
cmd="${cmd} --delete-excluded"
|
||||||
|
cmd="${cmd} --force"
|
||||||
|
cmd="${cmd} --ignore-errors"
|
||||||
|
cmd="${cmd} --log-file=${RSYNC_LOGFILE}"
|
||||||
|
cmd="${cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'"
|
||||||
|
|
||||||
|
# Rsync excludes
|
||||||
|
while read line ; do
|
||||||
|
cmd="${cmd} --exclude ${line}"
|
||||||
|
done < "${excludes_file}"
|
||||||
|
|
||||||
|
# Rsync local sources
|
||||||
|
cmd="${cmd} ${default_includes}"
|
||||||
|
while read line ; do
|
||||||
|
cmd="${cmd} ${line}"
|
||||||
|
done < "${includes_file}"
|
||||||
|
|
||||||
|
# Rsync remote destination
|
||||||
|
cmd="${cmd} root@${SSH_SERVER}:/var/backup/"
|
||||||
|
|
||||||
|
# output final command
|
||||||
|
echo "${cmd}"
|
||||||
|
}
|
||||||
sync_tasks() {
|
sync_tasks() {
|
||||||
n=0
|
n=0
|
||||||
server=""
|
server=""
|
||||||
|
@ -392,88 +515,30 @@ sync_tasks() {
|
||||||
SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
|
SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
|
||||||
SSH_PORT=$(echo "${server}" | cut -d':' -f2)
|
SSH_PORT=$(echo "${server}" | cut -d':' -f2)
|
||||||
|
|
||||||
if [ "${SYSTEM}" = "linux" ]; then
|
|
||||||
rep="/bin /boot /lib /opt /sbin /usr"
|
|
||||||
else
|
|
||||||
rep="/bsd /bin /sbin /usr"
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "START SYNC_TASKS - server=${server}"
|
log "START SYNC_TASKS - server=${server}"
|
||||||
|
|
||||||
update-evobackup-canary --who "${PROGNAME}"
|
# default paths, depending on system
|
||||||
|
if [ "${SYSTEM}" = "linux" ]; then
|
||||||
|
default_includes="/bin /boot /lib /opt /sbin /usr"
|
||||||
|
else
|
||||||
|
default_includes="/bsd /bin /sbin /usr"
|
||||||
|
fi
|
||||||
|
|
||||||
# Remote shell command
|
# reset Rsync log file
|
||||||
RSH_COMMAND="ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'"
|
if [ -n "$(command -v truncate)" ]; then
|
||||||
|
truncate -s 0 "${RSYNC_LOGFILE}"
|
||||||
|
else
|
||||||
|
printf "" > "${RSYNC_LOGFILE}"
|
||||||
|
fi
|
||||||
|
|
||||||
###################################################################
|
# Build the final Rsync command
|
||||||
# /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ #
|
rsync_cmd=$(build_rsync_cmd)
|
||||||
###################################################################
|
|
||||||
# DO NOT USE COMMENTS in rsync lines #
|
|
||||||
# DO NOT ADD WHITESPACES AFTER \ in rsync lines #
|
|
||||||
# It breaks the command and destroys data #
|
|
||||||
# => Only remove (or add) lines. #
|
|
||||||
###################################################################
|
|
||||||
|
|
||||||
# ignore check because we want it to split the different arguments to $rep
|
# … log it
|
||||||
# shellcheck disable=SC2086
|
log "SYNC_TASKS - Rsync command : ${rsync_cmd}"
|
||||||
rsync --archive \
|
|
||||||
--itemize-changes --stats --human-readable \
|
# … execute it
|
||||||
--relative --partial \
|
eval "${rsync_cmd}"
|
||||||
--delete --delete-excluded --force --ignore-errors \
|
|
||||||
--exclude "dev" \
|
|
||||||
--exclude "lost+found" \
|
|
||||||
--exclude ".nfs.*" \
|
|
||||||
--exclude "/usr/doc" \
|
|
||||||
--exclude "/usr/obj" \
|
|
||||||
--exclude "/usr/share/doc" \
|
|
||||||
--exclude "/usr/src" \
|
|
||||||
--exclude "/var/apt" \
|
|
||||||
--exclude "/var/cache" \
|
|
||||||
--exclude "/var/lib/amavis/amavisd.sock" \
|
|
||||||
--exclude "/var/lib/amavis/tmp" \
|
|
||||||
--exclude "/var/lib/clamav/*.tmp" \
|
|
||||||
--exclude "/var/lib/elasticsearch" \
|
|
||||||
--exclude "/var/lib/metche" \
|
|
||||||
--exclude "/var/lib/munin/*tmp*" \
|
|
||||||
--exclude "/var/db/munin/*.tmp" \
|
|
||||||
--exclude "/var/lib/mongodb" \
|
|
||||||
--exclude "/var/lib/mysql" \
|
|
||||||
--exclude "/var/lib/php5" \
|
|
||||||
--exclude "/var/lib/php/sessions" \
|
|
||||||
--exclude "/var/lib/postgres" \
|
|
||||||
--exclude "/var/lib/postgresql" \
|
|
||||||
--exclude "/var/lib/sympa" \
|
|
||||||
--exclude "/var/lock" \
|
|
||||||
--exclude "/var/run" \
|
|
||||||
--exclude "/var/spool/postfix" \
|
|
||||||
--exclude "/var/spool/smtpd" \
|
|
||||||
--exclude "/var/spool/squid" \
|
|
||||||
--exclude "/var/state" \
|
|
||||||
--exclude "/var/tmp" \
|
|
||||||
--exclude "lxc/*/rootfs/tmp" \
|
|
||||||
--exclude "lxc/*/rootfs/usr/doc" \
|
|
||||||
--exclude "lxc/*/rootfs/usr/obj" \
|
|
||||||
--exclude "lxc/*/rootfs/usr/share/doc" \
|
|
||||||
--exclude "lxc/*/rootfs/usr/src" \
|
|
||||||
--exclude "lxc/*/rootfs/var/apt" \
|
|
||||||
--exclude "lxc/*/rootfs/var/cache" \
|
|
||||||
--exclude "lxc/*/rootfs/var/lib/php5" \
|
|
||||||
--exclude "lxc/*/rootfs/var/lib/php/sessions" \
|
|
||||||
--exclude "lxc/*/rootfs/var/lock" \
|
|
||||||
--exclude "lxc/*/rootfs/var/log" \
|
|
||||||
--exclude "lxc/*/rootfs/var/run" \
|
|
||||||
--exclude "lxc/*/rootfs/var/state" \
|
|
||||||
--exclude "lxc/*/rootfs/var/tmp" \
|
|
||||||
--exclude "/home/mysqltmp" \
|
|
||||||
${rep} \
|
|
||||||
/etc \
|
|
||||||
/root \
|
|
||||||
/var \
|
|
||||||
/home \
|
|
||||||
/zzz_evobackup_canary \
|
|
||||||
-e "${RSH_COMMAND}" \
|
|
||||||
"root@${SSH_SERVER}:/var/backup/" \
|
|
||||||
| tail -30 >> "${LOGFILE}"
|
|
||||||
|
|
||||||
rsync_rc=$?
|
rsync_rc=$?
|
||||||
if [ ${rsync_rc} -ne 0 ]; then
|
if [ ${rsync_rc} -ne 0 ]; then
|
||||||
|
@ -481,6 +546,9 @@ sync_tasks() {
|
||||||
rc=201
|
rc=201
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Copy last lines of rsync log to the main log
|
||||||
|
tail -n 30 "${RSYNC_LOGFILE}" >> "${LOGFILE}"
|
||||||
|
|
||||||
log "STOP SYNC_TASKS - server=${server}"
|
log "STOP SYNC_TASKS - server=${server}"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -582,8 +650,16 @@ main() {
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
echo "$$" > "${PIDFILE}"
|
echo "$$" > "${PIDFILE}"
|
||||||
|
|
||||||
|
# Initialize a list of files to delete at exit
|
||||||
|
# Any file added to the list will also be deleted at exit
|
||||||
|
temp_files="${PIDFILE}"
|
||||||
|
|
||||||
# shellcheck disable=SC2064
|
# shellcheck disable=SC2064
|
||||||
trap "rm -f ${PIDFILE}" EXIT
|
trap "rm -f ${temp_files}" EXIT
|
||||||
|
|
||||||
|
# Update canary to keep track of each run
|
||||||
|
update-evobackup-canary --who "${PROGNAME}"
|
||||||
|
|
||||||
if [ "${LOCAL_TASKS}" = "1" ]; then
|
if [ "${LOCAL_TASKS}" = "1" ]; then
|
||||||
local_tasks
|
local_tasks
|
||||||
|
|
Loading…
Reference in a new issue