From 07f324bc7c6538e6d5fbbe2afce9c7342c219baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Mon, 18 Mar 2019 16:50:06 +0100 Subject: [PATCH 01/49] bkctld.conf: Add NODE by default --- bkctld.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/bkctld.conf b/bkctld.conf index ddcb872..d6307a7 100644 --- a/bkctld.conf +++ b/bkctld.conf @@ -13,3 +13,4 @@ #AUTHORIZED_KEYS='/root/.ssh/authorized_keys' #FIREWALL_RULES='' #LOGLEVEL=6 +#NODE='' From 3300c3df60bb2e5fe67cdfc3024c2bb10ce8da48 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 29 Mar 2019 18:20:32 +0100 Subject: [PATCH 02/49] Add a check-incs.sh script --- check-incs.sh | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 check-incs.sh diff --git a/check-incs.sh b/check-incs.sh new file mode 100644 index 0000000..3cd6c36 --- /dev/null +++ b/check-incs.sh @@ -0,0 +1,52 @@ +#!/bin/sh + +EVOBACKUP_CONFIGS="/etc/evobackup/*" + +relative_date() { + format=$(echo $1 | cut -d'.' -f1) + time_jump=$(echo $1 | cut -d'.' -f2) + + reference_date=$(date "${format}") + past_date=$(date --date "${reference_date} ${time_jump}" +"%Y-%m-%d") + + echo ${past_date} +} +inc_exists() { + ls -d /backup/incs/$1 > /dev/null 2>&1 +} +jail_exists() { + ls -d /backup/jails/$1 > /dev/null 2>&1 +} +# default return value is 0 (succes) +rc=0 +# loop for each configured jail +for file in ${EVOBACKUP_CONFIGS}; do + jail_name=$(basename $file) + # check if jail is present + if jail_exists ${jail_name}; then + # get jail last configuration date + jail_config_age=$(date --date "$(stat -c %y ${file})" +%s) + # loop for each line in jail configuration + for line in $(cat $file); do + # inc date in ISO format + inc_date=$(relative_date $line) + # inc date in seconds from epoch + inc_age=$(date --date "${inc_date}" +%s) + # check if the configuration changed after the inc date + if [ $jail_config_age -lt $inc_age ]; then + # Error if inc is not found + if ! inc_exists ${jail_name}/${inc_date}*; then + echo "ERROR: inc is missing \`${jail_name}/${inc_date}'" >&2 + rc=1 + fi + else + echo "INFO: no inc expected for ${inc_date} \`${jail_name}'" + fi + done + else + echo "ERROR: jail is missing \`${jail_name}'" >&2 + rc=1 + fi +done + +exit $rc From b5f1e13685fe19018561baaaab7535b9d33fa481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 31 Mar 2019 21:25:33 +0200 Subject: [PATCH 03/49] Add logic to have multiple available servers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SERVERS contains 1 or more servers to send backup files to. Each day a primary backup server is chosen. If it's not available the script falls back to the next server, and the next… --- zzz_evobackup | 68 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 6e3d2d7..927d26b 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -14,13 +14,11 @@ # Licence: AGPLv3 # # The following variables must be changed: -# SSH_PORT: The Port used for the ssh(1) jail on the backup server # MAIL: The email address to send notifications to. -# SRV: The hostname or IP address of the backup server. -# -# You must then uncomment the various -# examples that best suit your case +# SERVERS: The list of hosts (hostname or IP address) and SSH port +# to send backup files to. # +# You must then uncomment the various examples that best suit your case.# PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin @@ -46,20 +44,62 @@ fi echo "$$" > $PIDFILE trap "rm -f $PIDFILE" EXIT -# port SSH -SSH_PORT=2XXX - # email adress for notifications MAIL=jdoe@example.com # choose "linux" or "bsd" SYSTEM=$(uname | tr '[:upper:]' '[:lower:]') -# Variable to choose different backup server with date -NODE=$(($(date +%e) % 2)) +SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX" +SSH_CONNECT_TIMEOUT=10 -# serveur address for rsync -SRV="node$NODE.backup.example.com" +test_server() { + item=$1 + host=$(echo ${item} | cut -d':' -f1) + port=$(echo ${item} | cut -d':' -f2) + + # Test if the server is accepting connections + ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" ${host} -p ${port} -t "exit" + + if [ $? = 0 ]; then + return 0 + else + echo "Failed to connect to \`${item}' within ${SSH_CONNECT_TIMEOUT} seconds" >&2 + return 1 + fi +} +pick_server() { + inc=${1:-0} + list_length=$(echo "${SERVERS}" | wc -w) + + if [ "${inc}" -ge "${list_length}" ]; then + echo "No more server available" >&2 + return 1 + fi + + salt=$(hostname | cksum | cut -d' ' -f1) + item=$(echo $(( ($(date +%d) + salt + inc) % list_length ))) + field=$(( item + 1 )) + + echo "${SERVERS}" | cut -d' ' -f${field} +} + +n=0 +SERVER="" +while :; do + server=$(pick_server "${n}") + test $? = 0 || exit 2 + + if test_server "${server}"; then + SERVER="${server}" + break + else + n=$(( n + 1 )) + fi +done + +SSH_SERVER=$(echo $SERVER | cut -d':' -f1) +SSH_PORT=$(echo $SERVER | cut -d':' -f2) ## We use /home/backup : feel free to use your own dir mkdir -p -m 700 /home/backup @@ -249,8 +289,8 @@ rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial /var \ /home \ /srv \ - -e "ssh -p $SSH_PORT" \ - "root@$SRV:/var/backup/" \ + -e "ssh -p ${SSH_PORT}" \ + "root@${SSH_SERVER}:/var/backup/" \ | tail -30 >> /var/log/evobackup.log END=$(/bin/date +"%d-%m-%Y ; %H:%M") From ab57f64d93165b1606ce2804d779efd27cf36de6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 31 Mar 2019 21:26:52 +0200 Subject: [PATCH 04/49] Extract a LOCAL_BACKUP_DIR variable --- zzz_evobackup | 58 ++++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 927d26b..1028efe 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -102,16 +102,18 @@ SSH_SERVER=$(echo $SERVER | cut -d':' -f1) SSH_PORT=$(echo $SERVER | cut -d':' -f2) ## We use /home/backup : feel free to use your own dir -mkdir -p -m 700 /home/backup +LOCAL_BACKUP_DIR=/home/backup +# shellcheck disable=SC2174 +mkdir -p -m 700 ${LOCAL_BACKUP_DIR} ## OpenLDAP : example with slapcat -# slapcat -l /home/backup/ldap.bak +# slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak ### MySQL ## example with global and compressed mysqldump # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ -# --opt --all-databases --force --events --hex-blob | gzip --best > /home/backup/mysql.bak.gz +# --opt --all-databases --force --events --hex-blob | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz ## example with two dumps for each table (.sql/.txt) for all databases # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ @@ -143,7 +145,7 @@ mkdir -p -m 700 /home/backup # instance=$(echo "$instance"|awk '{ print $3 }') # if [ "$instance" != "3306" ] # then -# mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd > /home/backup/mysql.$instance.bak +# mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd > ${LOCAL_BACKUP_DIR}/mysql.$instance.bak # fi # done @@ -151,31 +153,31 @@ mkdir -p -m 700 /home/backup ## example with pg_dumpall (warning: you need space in ~postgres) # su - postgres -c "pg_dumpall > ~/pg.dump.bak" -# mv ~postgres/pg.dump.bak /home/backup/ +# mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/ ## another method with gzip directly piped # cd /var/lib/postgresql -# sudo -u postgres pg_dumpall | gzip > /home/backup/pg.dump.bak.gz +# sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz # cd - > /dev/null ## example with all tables from MYBASE excepts TABLE1 and TABLE2 -# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f /home/backup/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE +# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE ## example with only TABLE1 and TABLE2 from MYBASE -# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f /home/backup/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE +# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE ## MongoDB : example with mongodump ## don't forget to create use with read-only access ## > use admin ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } ) -# test -d /home/backup/mongodump/ && rm -rf /home/backup/mongodump/ -# mkdir -p -m 700 /home/backup/mongodump/ -# mongodump --quiet -u mongobackup -pPASS -o /home/backup/mongodump/ +# test -d ${LOCAL_BACKUP_DIR}/mongodump/ && rm -rf ${LOCAL_BACKUP_DIR}/mongodump/ +# mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/ +# mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/ # if [ $? -ne 0 ]; then # echo "Error with mongodump!" # fi ## Redis : example with copy .rdb file -# cp /var/lib/redis/dump.rdb /home/backup/ +# cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/ ## ElasticSearch, take a snapshot as a backup. ## Warning: You need to have a path.repo configured. @@ -199,48 +201,48 @@ mkdir -p -m 700 /home/backup # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log ## RabbitMQ : export config -#rabbitmqadmin export /home/backup/rabbitmq.config >> /var/log/evobackup.log +#rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> /var/log/evobackup.log ## Dump MBR / table partitions with dd and sfdisk ## Linux #for disk in $(ls /dev/[sv]d[a-z] 2>/dev/null); do # name=$(basename $disk) -# dd if=$disk of=/home/backup/MBR-$name bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" -# fdisk -l $disk > /home/backup/partitions-$name +# dd if=$disk of=${LOCAL_BACKUP_DIR}/MBR-$name bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" +# fdisk -l $disk > ${LOCAL_BACKUP_DIR}/partitions-$name #done -#cat /home/backup/partitions-* > /home/backup/partitions +#cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions ## OpenBSD -# disklabel sd0 > /home/backup/partitions +# disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions # backup MegaCli config -#megacli -CfgSave -f /home/backup/megacli_conf.dump -a0 >/dev/null +#megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null ## Dump system and kernel versions -uname -a > /home/backup/uname +uname -a > ${LOCAL_BACKUP_DIR}/uname ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do - mtr -r $addr > /home/backup/mtr-${addr} - traceroute -n $addr > /home/backup/traceroute-${addr} 2>&1 + mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr} + traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1 done ## Dump process with ps -ps auwwx >/home/backup/ps.out +ps auwwx >${LOCAL_BACKUP_DIR}/ps.out if [ "$SYSTEM" = "linux" ]; then ## Dump network connections with netstat - netstat -taupen >/home/backup/netstat.out + netstat -taupen > ${LOCAL_BACKUP_DIR}/netstat.out ## List Debian packages - dpkg -l >/home/backup/packages - dpkg --get-selections >/home/backup/packages.getselections - apt-cache dumpavail >/home/backup/packages.available + dpkg -l > ${LOCAL_BACKUP_DIR}/packages + dpkg --get-selections > ${LOCAL_BACKUP_DIR}/packages.getselections + apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available else ## Dump network connections with netstat - netstat -finet -atn >/home/backup/netstat.out + netstat -finet -atn > ${LOCAL_BACKUP_DIR}/netstat.out ## List OpenBSD packages - pkg_info -m >/home/backup/packages + pkg_info -m > ${LOCAL_BACKUP_DIR}/packages fi HOSTNAME=$(hostname) From b1f46bc86672e19faa6980037374c4bab0cb3020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 31 Mar 2019 21:27:10 +0200 Subject: [PATCH 05/49] Add braces around variables --- zzz_evobackup | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 1028efe..dee3cc0 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -229,7 +229,7 @@ done ## Dump process with ps ps auwwx >${LOCAL_BACKUP_DIR}/ps.out -if [ "$SYSTEM" = "linux" ]; then +if [ "${SYSTEM}" = "linux" ]; then ## Dump network connections with netstat netstat -taupen > ${LOCAL_BACKUP_DIR}/netstat.out @@ -249,7 +249,7 @@ HOSTNAME=$(hostname) BEGINNING=$(/bin/date +"%d-%m-%Y ; %H:%M") -if [ "$SYSTEM" = "linux" ]; then +if [ "${SYSTEM}" = "linux" ]; then rep="/bin /boot /lib /opt /sbin /usr" else rep="/bsd /bin /sbin /usr" @@ -285,7 +285,7 @@ rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial --exclude "/var/lib/clamav/*.tmp" \ --exclude "/home/mysqltmp" \ --exclude "/var/lib/php/sessions" \ - $rep \ + ${rep} \ /etc \ /root \ /var \ @@ -297,12 +297,12 @@ rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial END=$(/bin/date +"%d-%m-%Y ; %H:%M") -echo "EvoBackup - $HOSTNAME - START $BEGINNING" \ +echo "EvoBackup - ${HOSTNAME} - START ${BEGINNING}" \ >> /var/log/evobackup.log -echo "EvoBackup - $HOSTNAME - STOP $END" \ +echo "EvoBackup - ${HOSTNAME} - STOP ${END}" \ >> /var/log/evobackup.log tail -10 /var/log/evobackup.log | \ - mail -s "[info] EvoBackup - Client $HOSTNAME" \ - $MAIL + mail -s "[info] EvoBackup - Client ${HOSTNAME}" \ + ${MAIL} From 1376771e836154bcc5c4265055fd3a430751e3cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 31 Mar 2019 21:30:07 +0200 Subject: [PATCH 06/49] shellcheck fixes --- zzz_evobackup | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index dee3cc0..0fe145a 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -2,7 +2,7 @@ # # Script Evobackup client # See https://gitea.evolix.org/evolix/evobackup -# +# # Author: Gregory Colpart # Contributors: # Romain Dessort @@ -42,6 +42,7 @@ if [ -e $PIDFILE ]; then echo "$0 tourne encore (PID $pid). Processus killé" >&2 fi echo "$$" > $PIDFILE +# shellcheck disable=SC2064 trap "rm -f $PIDFILE" EXIT # email adress for notifications @@ -55,12 +56,12 @@ SSH_CONNECT_TIMEOUT=10 test_server() { item=$1 - host=$(echo ${item} | cut -d':' -f1) - port=$(echo ${item} | cut -d':' -f2) + host=$(echo "${item}" | cut -d':' -f1) + port=$(echo "${item}" | cut -d':' -f2) # Test if the server is accepting connections - ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" ${host} -p ${port} -t "exit" - + ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" "${host}" -p "${port}" -t "exit" + # shellcheck disable=SC2181 if [ $? = 0 ]; then return 0 else @@ -78,7 +79,7 @@ pick_server() { fi salt=$(hostname | cksum | cut -d' ' -f1) - item=$(echo $(( ($(date +%d) + salt + inc) % list_length ))) + item=$(( ($(date +%d) + salt + inc) % list_length )) field=$(( item + 1 )) echo "${SERVERS}" | cut -d' ' -f${field} @@ -98,11 +99,11 @@ while :; do fi done -SSH_SERVER=$(echo $SERVER | cut -d':' -f1) -SSH_PORT=$(echo $SERVER | cut -d':' -f2) +SSH_SERVER=$(echo "${SERVER}" | cut -d':' -f1) +SSH_PORT=$(echo "${SERVER}" | cut -d':' -f2) ## We use /home/backup : feel free to use your own dir -LOCAL_BACKUP_DIR=/home/backup +LOCAL_BACKUP_DIR="/home/backup" # shellcheck disable=SC2174 mkdir -p -m 700 ${LOCAL_BACKUP_DIR} @@ -285,7 +286,7 @@ rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial --exclude "/var/lib/clamav/*.tmp" \ --exclude "/home/mysqltmp" \ --exclude "/var/lib/php/sessions" \ - ${rep} \ + "${rep}" \ /etc \ /root \ /var \ From 34476ac7c7ed49ae3d31831f3e06c918386f558e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 31 Mar 2019 21:32:09 +0200 Subject: [PATCH 07/49] whitespaces --- zzz_evobackup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 0fe145a..a030796 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -207,9 +207,9 @@ mkdir -p -m 700 ${LOCAL_BACKUP_DIR} ## Dump MBR / table partitions with dd and sfdisk ## Linux #for disk in $(ls /dev/[sv]d[a-z] 2>/dev/null); do -# name=$(basename $disk) -# dd if=$disk of=${LOCAL_BACKUP_DIR}/MBR-$name bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" -# fdisk -l $disk > ${LOCAL_BACKUP_DIR}/partitions-$name +# name=$(basename $disk) +# dd if=$disk of=${LOCAL_BACKUP_DIR}/MBR-$name bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" +# fdisk -l $disk > ${LOCAL_BACKUP_DIR}/partitions-$name #done #cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions ## OpenBSD From e852827d9a6eaedb39d2e16c79281e3d42520ff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 31 Mar 2019 21:53:07 +0200 Subject: [PATCH 08/49] reorganize sections or the script --- zzz_evobackup | 114 ++++++++++++++++++++++++++++---------------------- 1 file changed, 64 insertions(+), 50 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index a030796..7adf6d9 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -13,12 +13,29 @@ # # Licence: AGPLv3 # -# The following variables must be changed: -# MAIL: The email address to send notifications to. -# SERVERS: The list of hosts (hostname or IP address) and SSH port -# to send backup files to. -# -# You must then uncomment the various examples that best suit your case.# +# /!\ DON'T FORGET TO SET "MAIL" and "SERVERS" VARIABLES + +##### Configuration ############################################################ + +# email adress for notifications +MAIL=jdoe@example.com + +# list of hosts (hostname or IP) and SSH port for Rsync +SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX" + +# timeout (in seconds) for the SSH test +SSH_CONNECT_TIMEOUT=10 + +## We use /home/backup : feel free to use your own dir +LOCAL_BACKUP_DIR="/home/backup" + +# You can set "linux" or "bsd" manually or let it choose automatically +SYSTEM=$(uname | tr '[:upper:]' '[:lower:]') + +##### SETUP AND FUNCTIONS ###################################################### + +# shellcheck disable=SC2174 +mkdir -p -m 700 ${LOCAL_BACKUP_DIR} PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin @@ -29,31 +46,6 @@ export LANG=C ## Force umask umask 077 -## Verify other evobackup process and kill if needed -PIDFILE=/var/run/evobackup.pid -if [ -e $PIDFILE ]; then - pid=$(cat "$PIDFILE") - # Killing the childs of evobackup. - for ppid in $(ps h --ppid "$pid" -o pid | tr -s '\n' ' '); do - kill -9 "$ppid"; - done - # Then kill the main PID. - kill -9 "$pid" - echo "$0 tourne encore (PID $pid). Processus killé" >&2 -fi -echo "$$" > $PIDFILE -# shellcheck disable=SC2064 -trap "rm -f $PIDFILE" EXIT - -# email adress for notifications -MAIL=jdoe@example.com - -# choose "linux" or "bsd" -SYSTEM=$(uname | tr '[:upper:]' '[:lower:]') - -SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX" -SSH_CONNECT_TIMEOUT=10 - test_server() { item=$1 host=$(echo "${item}" | cut -d':' -f1) @@ -85,27 +77,25 @@ pick_server() { echo "${SERVERS}" | cut -d' ' -f${field} } -n=0 -SERVER="" -while :; do - server=$(pick_server "${n}") - test $? = 0 || exit 2 +## Verify other evobackup process and kill if needed +PIDFILE=/var/run/evobackup.pid +if [ -e $PIDFILE ]; then + pid=$(cat "$PIDFILE") + # Killing the childs of evobackup. + for ppid in $(ps h --ppid "$pid" -o pid | tr -s '\n' ' '); do + kill -9 "$ppid"; + done + # Then kill the main PID. + kill -9 "$pid" + echo "$0 tourne encore (PID $pid). Processus killé" >&2 +fi +echo "$$" > $PIDFILE +# shellcheck disable=SC2064 +trap "rm -f $PIDFILE" EXIT - if test_server "${server}"; then - SERVER="${server}" - break - else - n=$(( n + 1 )) - fi -done +##### LOCAL BACKUP ############################################################# -SSH_SERVER=$(echo "${SERVER}" | cut -d':' -f1) -SSH_PORT=$(echo "${SERVER}" | cut -d':' -f2) - -## We use /home/backup : feel free to use your own dir -LOCAL_BACKUP_DIR="/home/backup" -# shellcheck disable=SC2174 -mkdir -p -m 700 ${LOCAL_BACKUP_DIR} +# You can comment or uncomment sections below to customize the backup ## OpenLDAP : example with slapcat # slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak @@ -246,6 +236,25 @@ else pkg_info -m > ${LOCAL_BACKUP_DIR}/packages fi +##### REMOTE BACKUP ############################################################ + +n=0 +server="" +while :; do + server=$(pick_server "${n}") + test $? = 0 || exit 2 + + if test_server "${server}"; then + break + else + server="" + n=$(( n + 1 )) + fi +done + +SSH_SERVER=$(echo "${server}" | cut -d':' -f1) +SSH_PORT=$(echo "${server}" | cut -d':' -f2) + HOSTNAME=$(hostname) BEGINNING=$(/bin/date +"%d-%m-%Y ; %H:%M") @@ -256,6 +265,9 @@ else rep="/bsd /bin /sbin /usr" fi +# /!\ DO NOT USE COMMENTS in the rsync command /!\ +# It breaks the command and destroys data, simply remove (or add) lines. + rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial \ --exclude "lost+found" \ --exclude ".nfs.*" \ @@ -298,6 +310,8 @@ rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial END=$(/bin/date +"%d-%m-%Y ; %H:%M") +##### REPORTING ################################################################ + echo "EvoBackup - ${HOSTNAME} - START ${BEGINNING}" \ >> /var/log/evobackup.log From dac06e0858dc92f220a660177420a2bfb3e7b21c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 31 Mar 2019 21:57:46 +0200 Subject: [PATCH 09/49] message in english --- zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 7adf6d9..521f597 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -87,7 +87,7 @@ if [ -e $PIDFILE ]; then done # Then kill the main PID. kill -9 "$pid" - echo "$0 tourne encore (PID $pid). Processus killé" >&2 + echo "$0 is still running (PID ${pid}). Process has been killed" >&2 fi echo "$$" > $PIDFILE # shellcheck disable=SC2064 From 92d6b8af8790cd074b87b05413e299d835faf38f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 31 Mar 2019 21:58:01 +0200 Subject: [PATCH 10/49] more braces and quotes --- zzz_evobackup | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 521f597..403e091 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -78,20 +78,20 @@ pick_server() { } ## Verify other evobackup process and kill if needed -PIDFILE=/var/run/evobackup.pid -if [ -e $PIDFILE ]; then - pid=$(cat "$PIDFILE") +PIDFILE="/var/run/evobackup.pid" +if [ -e "${PIDFILE}" ]; then + pid=$(cat "${PIDFILE}") # Killing the childs of evobackup. - for ppid in $(ps h --ppid "$pid" -o pid | tr -s '\n' ' '); do - kill -9 "$ppid"; + for ppid in $(ps h --ppid "${pid}" -o pid | tr -s '\n' ' '); do + kill -9 "${ppid}"; done # Then kill the main PID. - kill -9 "$pid" + kill -9 "${pid}" echo "$0 is still running (PID ${pid}). Process has been killed" >&2 fi -echo "$$" > $PIDFILE +echo "$$" > ${PIDFILE} # shellcheck disable=SC2064 -trap "rm -f $PIDFILE" EXIT +trap "rm -f ${PIDFILE}" EXIT ##### LOCAL BACKUP ############################################################# From abbb20e5d47e1388243728e09fcfa846aa92f9eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 31 Mar 2019 21:59:59 +0200 Subject: [PATCH 11/49] shorter lines for section titles --- zzz_evobackup | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 403e091..04d302a 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -15,7 +15,7 @@ # # /!\ DON'T FORGET TO SET "MAIL" and "SERVERS" VARIABLES -##### Configuration ############################################################ +##### Configuration ################################################### # email adress for notifications MAIL=jdoe@example.com @@ -32,7 +32,7 @@ LOCAL_BACKUP_DIR="/home/backup" # You can set "linux" or "bsd" manually or let it choose automatically SYSTEM=$(uname | tr '[:upper:]' '[:lower:]') -##### SETUP AND FUNCTIONS ###################################################### +##### SETUP AND FUNCTIONS ############################################# # shellcheck disable=SC2174 mkdir -p -m 700 ${LOCAL_BACKUP_DIR} @@ -93,7 +93,7 @@ echo "$$" > ${PIDFILE} # shellcheck disable=SC2064 trap "rm -f ${PIDFILE}" EXIT -##### LOCAL BACKUP ############################################################# +##### LOCAL BACKUP #################################################### # You can comment or uncomment sections below to customize the backup @@ -236,7 +236,7 @@ else pkg_info -m > ${LOCAL_BACKUP_DIR}/packages fi -##### REMOTE BACKUP ############################################################ +##### REMOTE BACKUP ################################################### n=0 server="" @@ -310,7 +310,7 @@ rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial END=$(/bin/date +"%d-%m-%Y ; %H:%M") -##### REPORTING ################################################################ +##### REPORTING ####################################################### echo "EvoBackup - ${HOSTNAME} - START ${BEGINNING}" \ >> /var/log/evobackup.log From ab3ed48f221a31bc2f523929cc37729fcdcc2f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 31 Mar 2019 22:09:53 +0200 Subject: [PATCH 12/49] don't quote this vaiable ; it's a list ! --- zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 04d302a..c760680 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -298,7 +298,7 @@ rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial --exclude "/var/lib/clamav/*.tmp" \ --exclude "/home/mysqltmp" \ --exclude "/var/lib/php/sessions" \ - "${rep}" \ + ${rep} \ /etc \ /root \ /var \ From d741041e4cc10932aaf04285dad2055243399422 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 1 Apr 2019 14:23:26 +0200 Subject: [PATCH 13/49] give SSH more time before failing --- zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index c760680..0e4fc3c 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -24,7 +24,7 @@ MAIL=jdoe@example.com SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX" # timeout (in seconds) for the SSH test -SSH_CONNECT_TIMEOUT=10 +SSH_CONNECT_TIMEOUT=30 ## We use /home/backup : feel free to use your own dir LOCAL_BACKUP_DIR="/home/backup" From c82e77f6eec8f166ec67d4e7f6f0805f7b247375 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 1 Apr 2019 14:37:57 +0200 Subject: [PATCH 14/49] Add documentation comments to test_server and pick_server functions --- zzz_evobackup | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 0e4fc3c..bd7aff2 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -46,8 +46,12 @@ export LANG=C ## Force umask umask 077 +# Call test_server with "HOST:PORT" string +# It will return with 0 if the server is reachable. +# It will return with 1 and a message on stderr if not. test_server() { item=$1 + # split HOST and PORT from the input string host=$(echo "${item}" | cut -d':' -f1) port=$(echo "${item}" | cut -d':' -f2) @@ -55,23 +59,32 @@ test_server() { ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" "${host}" -p "${port}" -t "exit" # shellcheck disable=SC2181 if [ $? = 0 ]; then + # SSH connection is OK return 0 else + # SSH connection failed echo "Failed to connect to \`${item}' within ${SSH_CONNECT_TIMEOUT} seconds" >&2 return 1 fi } +# Call pick_server with an optional positive integer to get the nth server in the list. pick_server() { - inc=${1:-0} + increment=${1:-0} list_length=$(echo "${SERVERS}" | wc -w) - if [ "${inc}" -ge "${list_length}" ]; then + if [ "${increment}" -ge "${list_length}" ]; then + # We've reached the end of the list echo "No more server available" >&2 return 1 fi + # A salt is useful to randomize the starting point in the list + # but stay identical each time it's called for a server (based on hostname). salt=$(hostname | cksum | cut -d' ' -f1) - item=$(( ($(date +%d) + salt + inc) % list_length )) + # Pick an integer between 0 and the length of the SERVERS list + # It changes each day + item=$(( ($(date +%d) + salt + increment) % list_length )) + # cut starts counting fields at 1, not 0. field=$(( item + 1 )) echo "${SERVERS}" | cut -d' ' -f${field} From c8e7977b274eb58f7b23cd57ab7407deb9f2b279 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 1 Apr 2019 14:38:48 +0200 Subject: [PATCH 15/49] revert variable extraction --- zzz_evobackup | 59 ++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index bd7aff2..633fb69 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -26,16 +26,13 @@ SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX" # timeout (in seconds) for the SSH test SSH_CONNECT_TIMEOUT=30 -## We use /home/backup : feel free to use your own dir -LOCAL_BACKUP_DIR="/home/backup" - # You can set "linux" or "bsd" manually or let it choose automatically SYSTEM=$(uname | tr '[:upper:]' '[:lower:]') ##### SETUP AND FUNCTIONS ############################################# # shellcheck disable=SC2174 -mkdir -p -m 700 ${LOCAL_BACKUP_DIR} +mkdir -p -m 700 /home/backup PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin @@ -111,13 +108,13 @@ trap "rm -f ${PIDFILE}" EXIT # You can comment or uncomment sections below to customize the backup ## OpenLDAP : example with slapcat -# slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak +# slapcat -l /home/backup/ldap.bak ### MySQL ## example with global and compressed mysqldump # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ -# --opt --all-databases --force --events --hex-blob | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz +# --opt --all-databases --force --events --hex-blob | gzip --best > /home/backup/mysql.bak.gz ## example with two dumps for each table (.sql/.txt) for all databases # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ @@ -149,7 +146,7 @@ trap "rm -f ${PIDFILE}" EXIT # instance=$(echo "$instance"|awk '{ print $3 }') # if [ "$instance" != "3306" ] # then -# mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd > ${LOCAL_BACKUP_DIR}/mysql.$instance.bak +# mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd > /home/backup/mysql.$instance.bak # fi # done @@ -157,31 +154,31 @@ trap "rm -f ${PIDFILE}" EXIT ## example with pg_dumpall (warning: you need space in ~postgres) # su - postgres -c "pg_dumpall > ~/pg.dump.bak" -# mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/ +# mv ~postgres/pg.dump.bak /home/backup/ ## another method with gzip directly piped # cd /var/lib/postgresql -# sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz +# sudo -u postgres pg_dumpall | gzip > /home/backup/pg.dump.bak.gz # cd - > /dev/null ## example with all tables from MYBASE excepts TABLE1 and TABLE2 -# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE +# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f /home/backup/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE ## example with only TABLE1 and TABLE2 from MYBASE -# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE +# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f /home/backup/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE ## MongoDB : example with mongodump ## don't forget to create use with read-only access ## > use admin ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } ) -# test -d ${LOCAL_BACKUP_DIR}/mongodump/ && rm -rf ${LOCAL_BACKUP_DIR}/mongodump/ -# mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/ -# mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/ +# test -d /home/backup/mongodump/ && rm -rf /home/backup/mongodump/ +# mkdir -p -m 700 /home/backup/mongodump/ +# mongodump --quiet -u mongobackup -pPASS -o /home/backup/mongodump/ # if [ $? -ne 0 ]; then # echo "Error with mongodump!" # fi ## Redis : example with copy .rdb file -# cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/ +# cp /var/lib/redis/dump.rdb /home/backup/ ## ElasticSearch, take a snapshot as a backup. ## Warning: You need to have a path.repo configured. @@ -205,48 +202,48 @@ trap "rm -f ${PIDFILE}" EXIT # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log ## RabbitMQ : export config -#rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> /var/log/evobackup.log +#rabbitmqadmin export /home/backup/rabbitmq.config >> /var/log/evobackup.log ## Dump MBR / table partitions with dd and sfdisk ## Linux #for disk in $(ls /dev/[sv]d[a-z] 2>/dev/null); do # name=$(basename $disk) -# dd if=$disk of=${LOCAL_BACKUP_DIR}/MBR-$name bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" -# fdisk -l $disk > ${LOCAL_BACKUP_DIR}/partitions-$name +# dd if=$disk of=/home/backup/MBR-$name bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" +# fdisk -l $disk > /home/backup/partitions-$name #done -#cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions +#cat /home/backup/partitions-* > /home/backup/partitions ## OpenBSD -# disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions +# disklabel sd0 > /home/backup/partitions # backup MegaCli config -#megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null +#megacli -CfgSave -f /home/backup/megacli_conf.dump -a0 >/dev/null ## Dump system and kernel versions -uname -a > ${LOCAL_BACKUP_DIR}/uname +uname -a > /home/backup/uname ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do - mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr} - traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1 + mtr -r ${addr} > /home/backup/mtr-${addr} + traceroute -n ${addr} > /home/backup/traceroute-${addr} 2>&1 done ## Dump process with ps -ps auwwx >${LOCAL_BACKUP_DIR}/ps.out +ps auwwx >/home/backup/ps.out if [ "${SYSTEM}" = "linux" ]; then ## Dump network connections with netstat - netstat -taupen > ${LOCAL_BACKUP_DIR}/netstat.out + netstat -taupen > /home/backup/netstat.out ## List Debian packages - dpkg -l > ${LOCAL_BACKUP_DIR}/packages - dpkg --get-selections > ${LOCAL_BACKUP_DIR}/packages.getselections - apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available + dpkg -l > /home/backup/packages + dpkg --get-selections > /home/backup/packages.getselections + apt-cache dumpavail > /home/backup/packages.available else ## Dump network connections with netstat - netstat -finet -atn > ${LOCAL_BACKUP_DIR}/netstat.out + netstat -finet -atn > /home/backup/netstat.out ## List OpenBSD packages - pkg_info -m > ${LOCAL_BACKUP_DIR}/packages + pkg_info -m > /home/backup/packages fi ##### REMOTE BACKUP ################################################### From 305655faa6b564be59cc54de7e28f6d1a00faca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Mon, 1 Apr 2019 22:14:49 +0200 Subject: [PATCH 16/49] revert some quotes and braces --- zzz_evobackup | 52 +++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 633fb69..999bc80 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -88,20 +88,20 @@ pick_server() { } ## Verify other evobackup process and kill if needed -PIDFILE="/var/run/evobackup.pid" -if [ -e "${PIDFILE}" ]; then - pid=$(cat "${PIDFILE}") +PIDFILE=/var/run/evobackup.pid +if [ -e $PIDFILE ]; then + pid=$(cat "$PIDFILE") # Killing the childs of evobackup. - for ppid in $(ps h --ppid "${pid}" -o pid | tr -s '\n' ' '); do - kill -9 "${ppid}"; + for ppid in $(ps h --ppid "$pid" -o pid | tr -s '\n' ' '); do + kill -9 "$ppid"; done # Then kill the main PID. - kill -9 "${pid}" - echo "$0 is still running (PID ${pid}). Process has been killed" >&2 + kill -9 "$pid" + echo "$0 is still running (PID $pid). Process has been killed" >&2 fi -echo "$$" > ${PIDFILE} +echo "$$" > $PIDFILE # shellcheck disable=SC2064 -trap "rm -f ${PIDFILE}" EXIT +trap "rm -f $PIDFILE" EXIT ##### LOCAL BACKUP #################################################### @@ -223,27 +223,27 @@ uname -a > /home/backup/uname ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do - mtr -r ${addr} > /home/backup/mtr-${addr} - traceroute -n ${addr} > /home/backup/traceroute-${addr} 2>&1 + mtr -r $addr > /home/backup/mtr-${addr} + traceroute -n $addr > /home/backup/traceroute-${addr} 2>&1 done ## Dump process with ps ps auwwx >/home/backup/ps.out -if [ "${SYSTEM}" = "linux" ]; then +if [ "$SYSTEM" = "linux" ]; then ## Dump network connections with netstat - netstat -taupen > /home/backup/netstat.out + netstat -taupen >/home/backup/netstat.out ## List Debian packages - dpkg -l > /home/backup/packages - dpkg --get-selections > /home/backup/packages.getselections - apt-cache dumpavail > /home/backup/packages.available + dpkg -l >/home/backup/packages + dpkg --get-selections >/home/backup/packages.getselections + apt-cache dumpavail >/home/backup/packages.available else ## Dump network connections with netstat - netstat -finet -atn > /home/backup/netstat.out + netstat -finet -atn >/home/backup/netstat.out ## List OpenBSD packages - pkg_info -m > /home/backup/packages + pkg_info -m >/home/backup/packages fi ##### REMOTE BACKUP ################################################### @@ -269,7 +269,7 @@ HOSTNAME=$(hostname) BEGINNING=$(/bin/date +"%d-%m-%Y ; %H:%M") -if [ "${SYSTEM}" = "linux" ]; then +if [ "$SYSTEM" = "linux" ]; then rep="/bin /boot /lib /opt /sbin /usr" else rep="/bsd /bin /sbin /usr" @@ -308,26 +308,26 @@ rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial --exclude "/var/lib/clamav/*.tmp" \ --exclude "/home/mysqltmp" \ --exclude "/var/lib/php/sessions" \ - ${rep} \ + $rep \ /etc \ /root \ /var \ /home \ /srv \ - -e "ssh -p ${SSH_PORT}" \ - "root@${SSH_SERVER}:/var/backup/" \ + -e "ssh -p $SSH_PORT" \ + "root@$SSH_SERVER:/var/backup/" \ | tail -30 >> /var/log/evobackup.log END=$(/bin/date +"%d-%m-%Y ; %H:%M") ##### REPORTING ####################################################### -echo "EvoBackup - ${HOSTNAME} - START ${BEGINNING}" \ +echo "EvoBackup - $HOSTNAME - START $BEGINNING" \ >> /var/log/evobackup.log -echo "EvoBackup - ${HOSTNAME} - STOP ${END}" \ +echo "EvoBackup - $HOSTNAME - STOP $END" \ >> /var/log/evobackup.log tail -10 /var/log/evobackup.log | \ - mail -s "[info] EvoBackup - Client ${HOSTNAME}" \ - ${MAIL} + mail -s "[info] EvoBackup - Client $HOSTNAME" \ + $MAIL From 7ddabb23ca3f5fc77cdf81d5cd23801d02ac4b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Wed, 3 Apr 2019 20:52:36 +0200 Subject: [PATCH 17/49] Extract a LOCAL_BACKUP_DIR variable --- zzz_evobackup | 59 +++++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 999bc80..05ffd46 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -26,13 +26,16 @@ SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX" # timeout (in seconds) for the SSH test SSH_CONNECT_TIMEOUT=30 +## We use /home/backup : feel free to use your own dir +LOCAL_BACKUP_DIR="/home/backup" + # You can set "linux" or "bsd" manually or let it choose automatically SYSTEM=$(uname | tr '[:upper:]' '[:lower:]') ##### SETUP AND FUNCTIONS ############################################# # shellcheck disable=SC2174 -mkdir -p -m 700 /home/backup +mkdir -p -m 700 ${LOCAL_BACKUP_DIR} PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin @@ -108,13 +111,13 @@ trap "rm -f $PIDFILE" EXIT # You can comment or uncomment sections below to customize the backup ## OpenLDAP : example with slapcat -# slapcat -l /home/backup/ldap.bak +# slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak ### MySQL ## example with global and compressed mysqldump # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ -# --opt --all-databases --force --events --hex-blob | gzip --best > /home/backup/mysql.bak.gz +# --opt --all-databases --force --events --hex-blob | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz ## example with two dumps for each table (.sql/.txt) for all databases # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ @@ -146,7 +149,7 @@ trap "rm -f $PIDFILE" EXIT # instance=$(echo "$instance"|awk '{ print $3 }') # if [ "$instance" != "3306" ] # then -# mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd > /home/backup/mysql.$instance.bak +# mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd > ${LOCAL_BACKUP_DIR}/mysql.$instance.bak # fi # done @@ -154,31 +157,31 @@ trap "rm -f $PIDFILE" EXIT ## example with pg_dumpall (warning: you need space in ~postgres) # su - postgres -c "pg_dumpall > ~/pg.dump.bak" -# mv ~postgres/pg.dump.bak /home/backup/ +# mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/ ## another method with gzip directly piped # cd /var/lib/postgresql -# sudo -u postgres pg_dumpall | gzip > /home/backup/pg.dump.bak.gz +# sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz # cd - > /dev/null ## example with all tables from MYBASE excepts TABLE1 and TABLE2 -# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f /home/backup/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE +# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE ## example with only TABLE1 and TABLE2 from MYBASE -# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f /home/backup/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE +# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE ## MongoDB : example with mongodump ## don't forget to create use with read-only access ## > use admin ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } ) -# test -d /home/backup/mongodump/ && rm -rf /home/backup/mongodump/ -# mkdir -p -m 700 /home/backup/mongodump/ -# mongodump --quiet -u mongobackup -pPASS -o /home/backup/mongodump/ +# test -d ${LOCAL_BACKUP_DIR}/mongodump/ && rm -rf ${LOCAL_BACKUP_DIR}/mongodump/ +# mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/ +# mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/ # if [ $? -ne 0 ]; then # echo "Error with mongodump!" # fi ## Redis : example with copy .rdb file -# cp /var/lib/redis/dump.rdb /home/backup/ +# cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/ ## ElasticSearch, take a snapshot as a backup. ## Warning: You need to have a path.repo configured. @@ -202,48 +205,48 @@ trap "rm -f $PIDFILE" EXIT # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log ## RabbitMQ : export config -#rabbitmqadmin export /home/backup/rabbitmq.config >> /var/log/evobackup.log +#rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> /var/log/evobackup.log ## Dump MBR / table partitions with dd and sfdisk ## Linux #for disk in $(ls /dev/[sv]d[a-z] 2>/dev/null); do # name=$(basename $disk) -# dd if=$disk of=/home/backup/MBR-$name bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" -# fdisk -l $disk > /home/backup/partitions-$name +# dd if=$disk of=${LOCAL_BACKUP_DIR}/MBR-$name bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" +# fdisk -l $disk > ${LOCAL_BACKUP_DIR}/partitions-$name #done -#cat /home/backup/partitions-* > /home/backup/partitions +#cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions ## OpenBSD -# disklabel sd0 > /home/backup/partitions +# disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions # backup MegaCli config -#megacli -CfgSave -f /home/backup/megacli_conf.dump -a0 >/dev/null +#megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null ## Dump system and kernel versions -uname -a > /home/backup/uname +uname -a > ${LOCAL_BACKUP_DIR}/uname ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do - mtr -r $addr > /home/backup/mtr-${addr} - traceroute -n $addr > /home/backup/traceroute-${addr} 2>&1 + mtr -r $addr > ${LOCAL_BACKUP_DIR}/mtr-${addr} + traceroute -n $addr > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1 done ## Dump process with ps -ps auwwx >/home/backup/ps.out +ps auwwx >${LOCAL_BACKUP_DIR}/ps.out if [ "$SYSTEM" = "linux" ]; then ## Dump network connections with netstat - netstat -taupen >/home/backup/netstat.out + netstat -taupen >${LOCAL_BACKUP_DIR}/netstat.out ## List Debian packages - dpkg -l >/home/backup/packages - dpkg --get-selections >/home/backup/packages.getselections - apt-cache dumpavail >/home/backup/packages.available + dpkg -l >${LOCAL_BACKUP_DIR}/packages + dpkg --get-selections >${LOCAL_BACKUP_DIR}/packages.getselections + apt-cache dumpavail >${LOCAL_BACKUP_DIR}/packages.available else ## Dump network connections with netstat - netstat -finet -atn >/home/backup/netstat.out + netstat -finet -atn >${LOCAL_BACKUP_DIR}/netstat.out ## List OpenBSD packages - pkg_info -m >/home/backup/packages + pkg_info -m >${LOCAL_BACKUP_DIR}/packages fi ##### REMOTE BACKUP ################################################### From 5719e2b028d70b497c081a3ac68210ba9ecc8bab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Wed, 3 Apr 2019 20:57:18 +0200 Subject: [PATCH 18/49] Add quotes and braces around variables --- zzz_evobackup | 52 +++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 05ffd46..bd7aff2 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -91,20 +91,20 @@ pick_server() { } ## Verify other evobackup process and kill if needed -PIDFILE=/var/run/evobackup.pid -if [ -e $PIDFILE ]; then - pid=$(cat "$PIDFILE") +PIDFILE="/var/run/evobackup.pid" +if [ -e "${PIDFILE}" ]; then + pid=$(cat "${PIDFILE}") # Killing the childs of evobackup. - for ppid in $(ps h --ppid "$pid" -o pid | tr -s '\n' ' '); do - kill -9 "$ppid"; + for ppid in $(ps h --ppid "${pid}" -o pid | tr -s '\n' ' '); do + kill -9 "${ppid}"; done # Then kill the main PID. - kill -9 "$pid" - echo "$0 is still running (PID $pid). Process has been killed" >&2 + kill -9 "${pid}" + echo "$0 is still running (PID ${pid}). Process has been killed" >&2 fi -echo "$$" > $PIDFILE +echo "$$" > ${PIDFILE} # shellcheck disable=SC2064 -trap "rm -f $PIDFILE" EXIT +trap "rm -f ${PIDFILE}" EXIT ##### LOCAL BACKUP #################################################### @@ -226,27 +226,27 @@ uname -a > ${LOCAL_BACKUP_DIR}/uname ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do - mtr -r $addr > ${LOCAL_BACKUP_DIR}/mtr-${addr} - traceroute -n $addr > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1 + mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr} + traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1 done ## Dump process with ps ps auwwx >${LOCAL_BACKUP_DIR}/ps.out -if [ "$SYSTEM" = "linux" ]; then +if [ "${SYSTEM}" = "linux" ]; then ## Dump network connections with netstat - netstat -taupen >${LOCAL_BACKUP_DIR}/netstat.out + netstat -taupen > ${LOCAL_BACKUP_DIR}/netstat.out ## List Debian packages - dpkg -l >${LOCAL_BACKUP_DIR}/packages - dpkg --get-selections >${LOCAL_BACKUP_DIR}/packages.getselections - apt-cache dumpavail >${LOCAL_BACKUP_DIR}/packages.available + dpkg -l > ${LOCAL_BACKUP_DIR}/packages + dpkg --get-selections > ${LOCAL_BACKUP_DIR}/packages.getselections + apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available else ## Dump network connections with netstat - netstat -finet -atn >${LOCAL_BACKUP_DIR}/netstat.out + netstat -finet -atn > ${LOCAL_BACKUP_DIR}/netstat.out ## List OpenBSD packages - pkg_info -m >${LOCAL_BACKUP_DIR}/packages + pkg_info -m > ${LOCAL_BACKUP_DIR}/packages fi ##### REMOTE BACKUP ################################################### @@ -272,7 +272,7 @@ HOSTNAME=$(hostname) BEGINNING=$(/bin/date +"%d-%m-%Y ; %H:%M") -if [ "$SYSTEM" = "linux" ]; then +if [ "${SYSTEM}" = "linux" ]; then rep="/bin /boot /lib /opt /sbin /usr" else rep="/bsd /bin /sbin /usr" @@ -311,26 +311,26 @@ rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial --exclude "/var/lib/clamav/*.tmp" \ --exclude "/home/mysqltmp" \ --exclude "/var/lib/php/sessions" \ - $rep \ + ${rep} \ /etc \ /root \ /var \ /home \ /srv \ - -e "ssh -p $SSH_PORT" \ - "root@$SSH_SERVER:/var/backup/" \ + -e "ssh -p ${SSH_PORT}" \ + "root@${SSH_SERVER}:/var/backup/" \ | tail -30 >> /var/log/evobackup.log END=$(/bin/date +"%d-%m-%Y ; %H:%M") ##### REPORTING ####################################################### -echo "EvoBackup - $HOSTNAME - START $BEGINNING" \ +echo "EvoBackup - ${HOSTNAME} - START ${BEGINNING}" \ >> /var/log/evobackup.log -echo "EvoBackup - $HOSTNAME - STOP $END" \ +echo "EvoBackup - ${HOSTNAME} - STOP ${END}" \ >> /var/log/evobackup.log tail -10 /var/log/evobackup.log | \ - mail -s "[info] EvoBackup - Client $HOSTNAME" \ - $MAIL + mail -s "[info] EvoBackup - Client ${HOSTNAME}" \ + ${MAIL} From 02a83c1a761b7e61ea43d7436e4c5aee00660dfa Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 8 Apr 2019 10:55:41 +0200 Subject: [PATCH 19/49] Remove the leading 0 from the date output --- zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 999bc80..3ca1c88 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -80,7 +80,7 @@ pick_server() { salt=$(hostname | cksum | cut -d' ' -f1) # Pick an integer between 0 and the length of the SERVERS list # It changes each day - item=$(( ($(date +%d) + salt + increment) % list_length )) + item=$(( ($(date +%e) + salt + increment) % list_length )) # cut starts counting fields at 1, not 0. field=$(( item + 1 )) From ead3cc5678756948784bc1ac78e3215678134e58 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 8 Apr 2019 11:09:11 +0200 Subject: [PATCH 20/49] extract date variable --- zzz_evobackup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 3ca1c88..7cb41a3 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -75,12 +75,14 @@ pick_server() { return 1 fi + # Extract the day of month, without leading 0 (which would give an octal based number) + today=$(date +%e) # A salt is useful to randomize the starting point in the list # but stay identical each time it's called for a server (based on hostname). salt=$(hostname | cksum | cut -d' ' -f1) # Pick an integer between 0 and the length of the SERVERS list # It changes each day - item=$(( ($(date +%e) + salt + increment) % list_length )) + item=$(( (today + salt + increment) % list_length )) # cut starts counting fields at 1, not 0. field=$(( item + 1 )) From ad82cc7b7af407b386aaf742b801a6bdd4619fee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Wed, 3 Apr 2019 21:02:26 +0200 Subject: [PATCH 21/49] replace netstat by ss on Linux --- zzz_evobackup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 7fb7181..82ab594 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -236,8 +236,8 @@ done ps auwwx >${LOCAL_BACKUP_DIR}/ps.out if [ "${SYSTEM}" = "linux" ]; then - ## Dump network connections with netstat - netstat -taupen > ${LOCAL_BACKUP_DIR}/netstat.out + ## Dump network connections with ss + ss -taupen > ${LOCAL_BACKUP_DIR}/netstat.out ## List Debian packages dpkg -l > ${LOCAL_BACKUP_DIR}/packages From a4ae0b84ed7b6566258be75c06454439774032e1 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 3 Jun 2019 14:56:35 +0200 Subject: [PATCH 22/49] zzz_evobackup : SSH timeout at 60 seconds by default --- zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 7fb7181..a8cffd9 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -24,7 +24,7 @@ MAIL=jdoe@example.com SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX" # timeout (in seconds) for the SSH test -SSH_CONNECT_TIMEOUT=30 +SSH_CONNECT_TIMEOUT=60 ## We use /home/backup : feel free to use your own dir LOCAL_BACKUP_DIR="/home/backup" From f9c9ab61134f0fce1528451e1288cac17587bc9c Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 3 Jun 2019 15:12:22 +0200 Subject: [PATCH 23/49] zzz_evobackup: add rsync server in logfile --- zzz_evobackup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 7fb7181..95dff5d 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -327,10 +327,10 @@ END=$(/bin/date +"%d-%m-%Y ; %H:%M") ##### REPORTING ####################################################### -echo "EvoBackup - ${HOSTNAME} - START ${BEGINNING}" \ +echo "EvoBackup - ${HOSTNAME} - START ${BEGINNING} ON ${SSH_SERVER}" \ >> /var/log/evobackup.log -echo "EvoBackup - ${HOSTNAME} - STOP ${END}" \ +echo "EvoBackup - ${HOSTNAME} - STOP ${END} ON ${SSH_SERVER}" \ >> /var/log/evobackup.log tail -10 /var/log/evobackup.log | \ From 3d3e235ecceba10c922f19476fb5f59b425f2fd5 Mon Sep 17 00:00:00 2001 From: Victor LABORIE Date: Wed, 5 Jun 2019 11:49:13 +0200 Subject: [PATCH 24/49] Make partitions dump by default on Linux --- zzz_evobackup | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index becd1c4..ffe9014 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -209,17 +209,6 @@ trap "rm -f ${PIDFILE}" EXIT ## RabbitMQ : export config #rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> /var/log/evobackup.log -## Dump MBR / table partitions with dd and sfdisk -## Linux -#for disk in $(ls /dev/[sv]d[a-z] 2>/dev/null); do -# name=$(basename $disk) -# dd if=$disk of=${LOCAL_BACKUP_DIR}/MBR-$name bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" -# fdisk -l $disk > ${LOCAL_BACKUP_DIR}/partitions-$name -#done -#cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions -## OpenBSD -# disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions - # backup MegaCli config #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null @@ -243,12 +232,23 @@ if [ "${SYSTEM}" = "linux" ]; then dpkg -l > ${LOCAL_BACKUP_DIR}/packages dpkg --get-selections > ${LOCAL_BACKUP_DIR}/packages.getselections apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available + + ## Dump MBR / table partitions + for disk in /dev/[sv]d[a-z]; do + name=$(basename "${disk}") + dd if="${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${name}" bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" + fdisk -l "${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${name}" + done + cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions else ## Dump network connections with netstat netstat -finet -atn > ${LOCAL_BACKUP_DIR}/netstat.out ## List OpenBSD packages pkg_info -m > ${LOCAL_BACKUP_DIR}/packages + + ## Dump MBR / table partitions + ##disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions fi ##### REMOTE BACKUP ################################################### From d0afc0089cc77bfc43f330ef2c952fb6b033f12c Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 11 Jun 2019 11:26:31 +0200 Subject: [PATCH 25/49] zzz_evobackup: better disks detection --- zzz_evobackup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index ffe9014..794dd38 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -234,7 +234,8 @@ if [ "${SYSTEM}" = "linux" ]; then apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available ## Dump MBR / table partitions - for disk in /dev/[sv]d[a-z]; do + disks=$(find /dev/ -regex '/dev/\([sv]d[a-z]\|nvme.*p\)[0-9]') + for disk in ${disks}; do name=$(basename "${disk}") dd if="${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${name}" bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" fdisk -l "${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${name}" From aa97d3ea88d40b62558f1aa6f28ca9be08debcca Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 24 Jun 2019 16:36:36 +0200 Subject: [PATCH 26/49] fix partitions list No need to look for all partitions, only disks are needed. --- zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 794dd38..7fc7d87 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -234,7 +234,7 @@ if [ "${SYSTEM}" = "linux" ]; then apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available ## Dump MBR / table partitions - disks=$(find /dev/ -regex '/dev/\([sv]d[a-z]\|nvme.*p\)[0-9]') + disks=$(find /dev/ -regex '/dev/\([sv]d[a-z]\|nvme[0-9]+n[0-9]+\)') for disk in ${disks}; do name=$(basename "${disk}") dd if="${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${name}" bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" From c3e9614dbbab8e7cbc9064377109b86a5caa3e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S?= Date: Wed, 10 Jul 2019 14:30:18 +0200 Subject: [PATCH 27/49] Adding LOGFILE variable This is way better than hardcoding log file path. --- zzz_evobackup | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 7fc7d87..24582ce 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -32,6 +32,10 @@ LOCAL_BACKUP_DIR="/home/backup" # You can set "linux" or "bsd" manually or let it choose automatically SYSTEM=$(uname | tr '[:upper:]' '[:lower:]') +# Change these 2 variables if you have more than one backup cron +PIDFILE="/var/run/evobackup.pid" +LOGFILE="/var/log/evobackup.log" + ##### SETUP AND FUNCTIONS ############################################# # shellcheck disable=SC2174 @@ -93,7 +97,6 @@ pick_server() { } ## Verify other evobackup process and kill if needed -PIDFILE="/var/run/evobackup.pid" if [ -e "${PIDFILE}" ]; then pid=$(cat "${PIDFILE}") # Killing the childs of evobackup. @@ -207,7 +210,7 @@ trap "rm -f ${PIDFILE}" EXIT # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log ## RabbitMQ : export config -#rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> /var/log/evobackup.log +#rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> $LOGFILE # backup MegaCli config #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null @@ -322,18 +325,18 @@ rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial /srv \ -e "ssh -p ${SSH_PORT}" \ "root@${SSH_SERVER}:/var/backup/" \ - | tail -30 >> /var/log/evobackup.log + | tail -30 >> $LOGFILE END=$(/bin/date +"%d-%m-%Y ; %H:%M") ##### REPORTING ####################################################### echo "EvoBackup - ${HOSTNAME} - START ${BEGINNING} ON ${SSH_SERVER}" \ - >> /var/log/evobackup.log + >> $LOGFILE echo "EvoBackup - ${HOSTNAME} - STOP ${END} ON ${SSH_SERVER}" \ - >> /var/log/evobackup.log + >> $LOGFILE -tail -10 /var/log/evobackup.log | \ +tail -10 $LOGFILE | \ mail -s "[info] EvoBackup - Client ${HOSTNAME}" \ ${MAIL} From c2cbcb23b44961944950ca5ec4825e72d53cc68b Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 22 Aug 2019 11:07:22 +0200 Subject: [PATCH 28/49] Print SSH connection errors only if no server is available. --- zzz_evobackup | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 24582ce..71d786c 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -50,6 +50,9 @@ export LANG=C ## Force umask umask 077 +## Initialize variable to store SSH connection errors +SERVERS_SSH_ERRORS="" + # Call test_server with "HOST:PORT" string # It will return with 0 if the server is reachable. # It will return with 1 and a message on stderr if not. @@ -67,7 +70,9 @@ test_server() { return 0 else # SSH connection failed - echo "Failed to connect to \`${item}' within ${SSH_CONNECT_TIMEOUT} seconds" >&2 + new_error=$(printf "Failed to connect to \`%s' within %s seconds" "${item}" "${SSH_CONNECT_TIMEOUT}") + SERVERS_SSH_ERRORS=$(printf "%s\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d') + return 1 fi } @@ -78,7 +83,10 @@ pick_server() { if [ "${increment}" -ge "${list_length}" ]; then # We've reached the end of the list - echo "No more server available" >&2 + new_error="No more server available" + SERVERS_SSH_ERRORS=$(printf "%s\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d') + + printf "%s\n" "${SERVERS_SSH_ERRORS}" >&2 return 1 fi From 6ede66fa5c7245040133a56a624a158dec463c47 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 22 Aug 2019 13:26:18 +0200 Subject: [PATCH 29/49] Use printf instead of echo where relevant --- zzz_evobackup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 71d786c..34e74f4 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -113,7 +113,7 @@ if [ -e "${PIDFILE}" ]; then done # Then kill the main PID. kill -9 "${pid}" - echo "$0 is still running (PID ${pid}). Process has been killed" >&2 + printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\n" >&2 fi echo "$$" > ${PIDFILE} # shellcheck disable=SC2064 @@ -339,10 +339,10 @@ END=$(/bin/date +"%d-%m-%Y ; %H:%M") ##### REPORTING ####################################################### -echo "EvoBackup - ${HOSTNAME} - START ${BEGINNING} ON ${SSH_SERVER}" \ +printf "EvoBackup - %s - START %s ON %s\n" "${HOSTNAME}" "${BEGINNING}" "${SSH_SERVER}" \ >> $LOGFILE -echo "EvoBackup - ${HOSTNAME} - STOP ${END} ON ${SSH_SERVER}" \ +printf "EvoBackup - %s - STOP %s ON %s\n" "${HOSTNAME}" "${END}" "${SSH_SERVER}" \ >> $LOGFILE tail -10 $LOGFILE | \ From 242153b4728e43c3f4af76888dd9ffefdf256ffa Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 22 Aug 2019 14:50:20 +0200 Subject: [PATCH 30/49] Make local and sync tasks skipable. --- zzz_evobackup | 344 ++++++++++++++++++++++++++------------------------ 1 file changed, 178 insertions(+), 166 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 34e74f4..95cdf74 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -36,8 +36,14 @@ SYSTEM=$(uname | tr '[:upper:]' '[:lower:]') PIDFILE="/var/run/evobackup.pid" LOGFILE="/var/log/evobackup.log" +## Enable/Disable tasks +LOCAL_TASKS=1 +SYNC_TASKS=1 + ##### SETUP AND FUNCTIONS ############################################# +BEGINNING=$(/bin/date +"%d-%m-%Y ; %H:%M") + # shellcheck disable=SC2174 mkdir -p -m 700 ${LOCAL_BACKUP_DIR} @@ -121,146 +127,149 @@ trap "rm -f ${PIDFILE}" EXIT ##### LOCAL BACKUP #################################################### -# You can comment or uncomment sections below to customize the backup +if [ "${LOCAL_TASKS}" = "1" ]; then + # You can comment or uncomment sections below to customize the backup -## OpenLDAP : example with slapcat -# slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak + ## OpenLDAP : example with slapcat + # slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak -### MySQL + ### MySQL -## example with global and compressed mysqldump -# mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ -# --opt --all-databases --force --events --hex-blob | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz + ## example with global and compressed mysqldump + # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ + # --opt --all-databases --force --events --hex-blob | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz -## example with two dumps for each table (.sql/.txt) for all databases -# for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ -# | egrep -v "^(Database|information_schema|performance_schema|sys)" ); \ -# do mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump ; \ -# mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \ -# --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i; done + ## example with two dumps for each table (.sql/.txt) for all databases + # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ + # | egrep -v "^(Database|information_schema|performance_schema|sys)" ); \ + # do mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump ; \ + # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \ + # --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i; done -## example with compressed SQL dump for each databases -# mkdir -p -m 700 /home/mysqldump/ -# for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \ -# | egrep -v "^(Database|information_schema|performance_schema|sys)"); do -# mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i | gzip --best > /home/mysqldump/${i}.sql.gz -# done + ## example with compressed SQL dump for each databases + # mkdir -p -m 700 /home/mysqldump/ + # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \ + # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do + # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i | gzip --best > /home/mysqldump/${i}.sql.gz + # done -## example with *one* uncompressed SQL dump for *one* database (MYBASE) -# mkdir -p -m 700 /home/mysqldump/MYBASE -# chown -RL mysql /home/mysqldump/ -# mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -Q \ -# --opt --events --hex-blob --skip-comments -T /home/mysqldump/MYBASE MYBASE + ## example with *one* uncompressed SQL dump for *one* database (MYBASE) + # mkdir -p -m 700 /home/mysqldump/MYBASE + # chown -RL mysql /home/mysqldump/ + # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -Q \ + # --opt --events --hex-blob --skip-comments -T /home/mysqldump/MYBASE MYBASE -## example with mysqlhotcopy -# mkdir -p -m 700 /home/mysqlhotcopy/ -# mysqlhotcopy BASE /home/mysqlhotcopy/ + ## example with mysqlhotcopy + # mkdir -p -m 700 /home/mysqlhotcopy/ + # mysqlhotcopy BASE /home/mysqlhotcopy/ -## example for multiples MySQL instances -# mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3) -# grep -E "^port\s*=\s*\d*" /etc/mysql/my.cnf |while read instance; do -# instance=$(echo "$instance"|awk '{ print $3 }') -# if [ "$instance" != "3306" ] -# then -# mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd > ${LOCAL_BACKUP_DIR}/mysql.$instance.bak -# fi -# done + ## example for multiples MySQL instances + # mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3) + # grep -E "^port\s*=\s*\d*" /etc/mysql/my.cnf |while read instance; do + # instance=$(echo "$instance"|awk '{ print $3 }') + # if [ "$instance" != "3306" ] + # then + # mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd > ${LOCAL_BACKUP_DIR}/mysql.$instance.bak + # fi + # done -### PostgreSQL + ### PostgreSQL -## example with pg_dumpall (warning: you need space in ~postgres) -# su - postgres -c "pg_dumpall > ~/pg.dump.bak" -# mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/ -## another method with gzip directly piped -# cd /var/lib/postgresql -# sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz -# cd - > /dev/null + ## example with pg_dumpall (warning: you need space in ~postgres) + # su - postgres -c "pg_dumpall > ~/pg.dump.bak" + # mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/ + ## another method with gzip directly piped + # cd /var/lib/postgresql + # sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz + # cd - > /dev/null -## example with all tables from MYBASE excepts TABLE1 and TABLE2 -# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE + ## example with all tables from MYBASE excepts TABLE1 and TABLE2 + # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE -## example with only TABLE1 and TABLE2 from MYBASE -# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE + ## example with only TABLE1 and TABLE2 from MYBASE + # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE -## MongoDB : example with mongodump -## don't forget to create use with read-only access -## > use admin -## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } ) -# test -d ${LOCAL_BACKUP_DIR}/mongodump/ && rm -rf ${LOCAL_BACKUP_DIR}/mongodump/ -# mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/ -# mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/ -# if [ $? -ne 0 ]; then -# echo "Error with mongodump!" -# fi + ## MongoDB : example with mongodump + ## don't forget to create use with read-only access + ## > use admin + ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } ) + # test -d ${LOCAL_BACKUP_DIR}/mongodump/ && rm -rf ${LOCAL_BACKUP_DIR}/mongodump/ + # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/ + # mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/ + # if [ $? -ne 0 ]; then + # echo "Error with mongodump!" + # fi -## Redis : example with copy .rdb file -# cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/ + ## Redis : example with copy .rdb file + # cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/ -## ElasticSearch, take a snapshot as a backup. -## Warning: You need to have a path.repo configured. -## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes -# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log -# curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log -## Clustered version here -## It basically the same thing except that you need to check that NFS is mounted -# if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2' -# then -# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log -# curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log -# else -# echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.' -# fi -## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by : -# for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do -# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}' -# done -# date=$(date +%F) -# curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log + ## ElasticSearch, take a snapshot as a backup. + ## Warning: You need to have a path.repo configured. + ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes + # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log + # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log + ## Clustered version here + ## It basically the same thing except that you need to check that NFS is mounted + # if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2' + # then + # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log + # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log + # else + # echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.' + # fi + ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by : + # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do + # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}' + # done + # date=$(date +%F) + # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log -## RabbitMQ : export config -#rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> $LOGFILE + ## RabbitMQ : export config + #rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> $LOGFILE -# backup MegaCli config -#megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null + # backup MegaCli config + #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null -## Dump system and kernel versions -uname -a > ${LOCAL_BACKUP_DIR}/uname + ## Dump system and kernel versions + uname -a > ${LOCAL_BACKUP_DIR}/uname -## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) -for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do - mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr} - traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1 -done - -## Dump process with ps -ps auwwx >${LOCAL_BACKUP_DIR}/ps.out - -if [ "${SYSTEM}" = "linux" ]; then - ## Dump network connections with ss - ss -taupen > ${LOCAL_BACKUP_DIR}/netstat.out - - ## List Debian packages - dpkg -l > ${LOCAL_BACKUP_DIR}/packages - dpkg --get-selections > ${LOCAL_BACKUP_DIR}/packages.getselections - apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available - - ## Dump MBR / table partitions - disks=$(find /dev/ -regex '/dev/\([sv]d[a-z]\|nvme[0-9]+n[0-9]+\)') - for disk in ${disks}; do - name=$(basename "${disk}") - dd if="${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${name}" bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" - fdisk -l "${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${name}" + ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) + for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do + mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr} + traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1 done - cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions -else - ## Dump network connections with netstat - netstat -finet -atn > ${LOCAL_BACKUP_DIR}/netstat.out - ## List OpenBSD packages - pkg_info -m > ${LOCAL_BACKUP_DIR}/packages + ## Dump process with ps + ps auwwx >${LOCAL_BACKUP_DIR}/ps.out + + if [ "${SYSTEM}" = "linux" ]; then + ## Dump network connections with ss + ss -taupen > ${LOCAL_BACKUP_DIR}/netstat.out + + ## List Debian packages + dpkg -l > ${LOCAL_BACKUP_DIR}/packages + dpkg --get-selections > ${LOCAL_BACKUP_DIR}/packages.getselections + apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available + + ## Dump MBR / table partitions + disks=$(find /dev/ -regex '/dev/\([sv]d[a-z]\|nvme[0-9]+n[0-9]+\)') + for disk in ${disks}; do + name=$(basename "${disk}") + dd if="${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${name}" bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" + fdisk -l "${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${name}" + done + cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions + else + ## Dump network connections with netstat + netstat -finet -atn > ${LOCAL_BACKUP_DIR}/netstat.out + + ## List OpenBSD packages + pkg_info -m > ${LOCAL_BACKUP_DIR}/packages + + ## Dump MBR / table partitions + ##disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions + fi - ## Dump MBR / table partitions - ##disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions fi ##### REMOTE BACKUP ################################################### @@ -284,66 +293,69 @@ SSH_PORT=$(echo "${server}" | cut -d':' -f2) HOSTNAME=$(hostname) -BEGINNING=$(/bin/date +"%d-%m-%Y ; %H:%M") - if [ "${SYSTEM}" = "linux" ]; then rep="/bin /boot /lib /opt /sbin /usr" else rep="/bsd /bin /sbin /usr" fi -# /!\ DO NOT USE COMMENTS in the rsync command /!\ -# It breaks the command and destroys data, simply remove (or add) lines. -rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial \ - --exclude "lost+found" \ - --exclude ".nfs.*" \ - --exclude "/var/log" \ - --exclude "/var/log/evobackup*" \ - --exclude "/var/lib/mysql" \ - --exclude "/var/lib/postgres" \ - --exclude "/var/lib/postgresql" \ - --exclude "/var/lib/sympa" \ - --exclude "/var/lib/metche" \ - --exclude "/var/run" \ - --exclude "/var/lock" \ - --exclude "/var/state" \ - --exclude "/var/apt" \ - --exclude "/var/cache" \ - --exclude "/usr/src" \ - --exclude "/usr/doc" \ - --exclude "/usr/share/doc" \ - --exclude "/usr/obj" \ - --exclude "dev" \ - --exclude "/var/spool/postfix" \ - --exclude "/var/lib/amavis/amavisd.sock" \ - --exclude "/var/lib/munin/*tmp*" \ - --exclude "/var/lib/php5" \ - --exclude "/var/spool/squid" \ - --exclude "/var/lib/elasticsearch" \ - --exclude "/var/lib/amavis/tmp" \ - --exclude "/var/lib/clamav/*.tmp" \ - --exclude "/home/mysqltmp" \ - --exclude "/var/lib/php/sessions" \ - ${rep} \ - /etc \ - /root \ - /var \ - /home \ - /srv \ - -e "ssh -p ${SSH_PORT}" \ - "root@${SSH_SERVER}:/var/backup/" \ - | tail -30 >> $LOGFILE +if [ "${SYNC_TASKS}" = "1" ]; then + # /!\ DO NOT USE COMMENTS in the rsync command /!\ + # It breaks the command and destroys data, simply remove (or add) lines. -END=$(/bin/date +"%d-%m-%Y ; %H:%M") + rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial \ + --exclude "lost+found" \ + --exclude ".nfs.*" \ + --exclude "/var/log" \ + --exclude "/var/log/evobackup*" \ + --exclude "/var/lib/mysql" \ + --exclude "/var/lib/postgres" \ + --exclude "/var/lib/postgresql" \ + --exclude "/var/lib/sympa" \ + --exclude "/var/lib/metche" \ + --exclude "/var/run" \ + --exclude "/var/lock" \ + --exclude "/var/state" \ + --exclude "/var/apt" \ + --exclude "/var/cache" \ + --exclude "/usr/src" \ + --exclude "/usr/doc" \ + --exclude "/usr/share/doc" \ + --exclude "/usr/obj" \ + --exclude "dev" \ + --exclude "/var/spool/postfix" \ + --exclude "/var/lib/amavis/amavisd.sock" \ + --exclude "/var/lib/munin/*tmp*" \ + --exclude "/var/lib/php5" \ + --exclude "/var/spool/squid" \ + --exclude "/var/lib/elasticsearch" \ + --exclude "/var/lib/amavis/tmp" \ + --exclude "/var/lib/clamav/*.tmp" \ + --exclude "/home/mysqltmp" \ + --exclude "/var/lib/php/sessions" \ + ${rep} \ + /etc \ + /root \ + /var \ + /home \ + /srv \ + -e "ssh -p ${SSH_PORT}" \ + "root@${SSH_SERVER}:/var/backup/" \ + | tail -30 >> $LOGFILE +fi ##### REPORTING ####################################################### -printf "EvoBackup - %s - START %s ON %s\n" "${HOSTNAME}" "${BEGINNING}" "${SSH_SERVER}" \ - >> $LOGFILE +END=$(/bin/date +"%d-%m-%Y ; %H:%M") -printf "EvoBackup - %s - STOP %s ON %s\n" "${HOSTNAME}" "${END}" "${SSH_SERVER}" \ - >> $LOGFILE +printf "EvoBackup - %s - START %s ON %s (LOCAL_TASKS=%s SYNC_TASKS=%s)\n" \ + "${HOSTNAME}" "${BEGINNING}" "${SSH_SERVER}" "${LOCAL_TASKS}" "${SYNC_TASKS}" \ + >> $LOGFILE + +printf "EvoBackup - %s - STOP %s ON %s (LOCAL_TASKS=%s SYNC_TASKS=%s)\n" \ + "${HOSTNAME}" "${END}" "${SSH_SERVER}" "${LOCAL_TASKS}" "${SYNC_TASKS}" \ + >> $LOGFILE tail -10 $LOGFILE | \ mail -s "[info] EvoBackup - Client ${HOSTNAME}" \ From 24481ca37e21637bb0e6cdbd30b3af97cfd7c1f7 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 23 Aug 2019 12:04:57 +0200 Subject: [PATCH 31/49] Same timeout for Rsync than SSH --- zzz_evobackup | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 95cdf74..6fb40fd 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -304,6 +304,9 @@ if [ "${SYNC_TASKS}" = "1" ]; then # /!\ DO NOT USE COMMENTS in the rsync command /!\ # It breaks the command and destroys data, simply remove (or add) lines. + # Remote shell command + RSH_COMMAND="ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'" + rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial \ --exclude "lost+found" \ --exclude ".nfs.*" \ @@ -340,7 +343,7 @@ if [ "${SYNC_TASKS}" = "1" ]; then /var \ /home \ /srv \ - -e "ssh -p ${SSH_PORT}" \ + -e "${RSH_COMMAND}" \ "root@${SSH_SERVER}:/var/backup/" \ | tail -30 >> $LOGFILE fi From 55e9c0dd6382077b6a4e1a4a08a1b0ee4c03b50b Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 23 Aug 2019 12:05:22 +0200 Subject: [PATCH 32/49] Tasks switch are configurable from envirnoment variables too. --- zzz_evobackup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 6fb40fd..f262349 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -37,8 +37,8 @@ PIDFILE="/var/run/evobackup.pid" LOGFILE="/var/log/evobackup.log" ## Enable/Disable tasks -LOCAL_TASKS=1 -SYNC_TASKS=1 +LOCAL_TASKS=${LOCAL_TASKS:-1} +SYNC_TASKS=${SYNC_TASKS:-1} ##### SETUP AND FUNCTIONS ############################################# From d7b221796e7937dc47ec0b500ad0ab005afc6c29 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 23 Aug 2019 14:26:31 +0200 Subject: [PATCH 33/49] =?UTF-8?q?30=20secondes=20suffisent=20par=20d=C3=A9?= =?UTF-8?q?faut?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- zzz_evobackup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index f262349..9657c31 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -23,8 +23,8 @@ MAIL=jdoe@example.com # list of hosts (hostname or IP) and SSH port for Rsync SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX" -# timeout (in seconds) for the SSH test -SSH_CONNECT_TIMEOUT=60 +# timeout (in seconds) for SSH connections +SSH_CONNECT_TIMEOUT=30 ## We use /home/backup : feel free to use your own dir LOCAL_BACKUP_DIR="/home/backup" From 35f343423749ae987fd2e852195a917959203d08 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 3 Sep 2019 10:26:38 +0200 Subject: [PATCH 34/49] Use lsblk to find actual disks --- zzz_evobackup | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 9657c31..438d24d 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -252,11 +252,10 @@ if [ "${LOCAL_TASKS}" = "1" ]; then apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available ## Dump MBR / table partitions - disks=$(find /dev/ -regex '/dev/\([sv]d[a-z]\|nvme[0-9]+n[0-9]+\)') + disks=$(lsblk -l | grep disk | awk '{print $1}') for disk in ${disks}; do - name=$(basename "${disk}") - dd if="${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${name}" bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)" - fdisk -l "${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${name}" + dd if="/dev/${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)" + fdisk -l "/disk/${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${disk}" done cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions else From e2156ad41a8b21407d7164fa46fb55b8ce1536b4 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 4 Sep 2019 13:16:09 +0200 Subject: [PATCH 35/49] Fix zzz_evobackup for fdisk command --- zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 438d24d..026a106 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -255,7 +255,7 @@ if [ "${LOCAL_TASKS}" = "1" ]; then disks=$(lsblk -l | grep disk | awk '{print $1}') for disk in ${disks}; do dd if="/dev/${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)" - fdisk -l "/disk/${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${disk}" + fdisk -l "/dev/${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${disk}" done cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions else From 04296895dae0601c7cc8a5781360247c6c0078e9 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 5 Sep 2019 11:15:25 +0200 Subject: [PATCH 36/49] exclude drbd disks from list --- zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 026a106..c3c3586 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -252,7 +252,7 @@ if [ "${LOCAL_TASKS}" = "1" ]; then apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available ## Dump MBR / table partitions - disks=$(lsblk -l | grep disk | awk '{print $1}') + disks=$(lsblk -l | grep disk | grep -v drbd | awk '{print $1}') for disk in ${disks}; do dd if="/dev/${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)" fdisk -l "/dev/${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${disk}" From 7eecbe3745e5f4590d2b6de9a57a9867eb11fa21 Mon Sep 17 00:00:00 2001 From: Tristan PILAT Date: Thu, 5 Sep 2019 11:41:56 +0200 Subject: [PATCH 37/49] Replace ps for pgrep to list evobackup child procs The command being used to identify evobackup child processes isn't supported on OpenBSD. The pgrep command is supported on both Linux and OpenBSD. --- zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 438d24d..74cc642 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -114,7 +114,7 @@ pick_server() { if [ -e "${PIDFILE}" ]; then pid=$(cat "${PIDFILE}") # Killing the childs of evobackup. - for ppid in $(ps h --ppid "${pid}" -o pid | tr -s '\n' ' '); do + for ppid in $(pgrep -P "${pid}"); do kill -9 "${ppid}"; done # Then kill the main PID. From 6b26d7fc5323caff583a970a2a6384c7c12e72e0 Mon Sep 17 00:00:00 2001 From: Gregory Colpart Date: Mon, 16 Sep 2019 14:15:50 +0200 Subject: [PATCH 38/49] add command to dump Unix rights --- zzz_evobackup | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zzz_evobackup b/zzz_evobackup index ec7d141..b3744b9 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -269,6 +269,12 @@ if [ "${LOCAL_TASKS}" = "1" ]; then ##disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions fi + ## Dump rights + #getfacl -R /var > ${LOCAL_BACKUP_DIR}/rights-var.txt + #getfacl -R /etc > ${LOCAL_BACKUP_DIR}/rights-etc.txt + #getfacl -R /usr > ${LOCAL_BACKUP_DIR}/rights-usr.txt + #getfacl -R /home > ${LOCAL_BACKUP_DIR}/rights-home.txt + fi ##### REMOTE BACKUP ################################################### From ddba0942e3e7afc3d7cd33d8e8b4f5f1991d5a3a Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 23 Sep 2019 18:06:00 +0200 Subject: [PATCH 39/49] Add variable to toggle the servers fallback --- zzz_evobackup | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index b3744b9..9732cbb 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -23,6 +23,9 @@ MAIL=jdoe@example.com # list of hosts (hostname or IP) and SSH port for Rsync SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX" +# Should we fallback on servers when the first is unreachable ? +SERVERS_FALLBACK=${SERVERS_FALLBACK:-1} + # timeout (in seconds) for SSH connections SSH_CONNECT_TIMEOUT=30 @@ -281,17 +284,23 @@ fi n=0 server="" -while :; do - server=$(pick_server "${n}") - test $? = 0 || exit 2 +if [ "${SERVERS_FALLBACK}" = "1" ]; then + # We try to find a suitable server + while :; do + server=$(pick_server "${n}") + test $? = 0 || exit 2 - if test_server "${server}"; then - break - else - server="" - n=$(( n + 1 )) - fi -done + if test_server "${server}"; then + break + else + server="" + n=$(( n + 1 )) + fi + done +else + # we force the server + server=$(pick_server "${n}") +fi SSH_SERVER=$(echo "${server}" | cut -d':' -f1) SSH_PORT=$(echo "${server}" | cut -d':' -f2) From e83f94eaad2a879b8b5cd1921b54f7c626fec6be Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 23 Sep 2019 18:06:27 +0200 Subject: [PATCH 40/49] make SSH_CONNECT_TIMEOUT configurable from outside --- zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 9732cbb..e2a20a0 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -27,7 +27,7 @@ SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX" SERVERS_FALLBACK=${SERVERS_FALLBACK:-1} # timeout (in seconds) for SSH connections -SSH_CONNECT_TIMEOUT=30 +SSH_CONNECT_TIMEOUT=${SSH_CONNECT_TIMEOUT:-30} ## We use /home/backup : feel free to use your own dir LOCAL_BACKUP_DIR="/home/backup" From 20237243709f768539a01c404df781269cd7a9f5 Mon Sep 17 00:00:00 2001 From: Gregory Colpart Date: Wed, 25 Sep 2019 15:10:47 +0200 Subject: [PATCH 41/49] Add dump iptables / pf --- zzz_evobackup | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zzz_evobackup b/zzz_evobackup index e2a20a0..d0225f0 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -261,6 +261,10 @@ if [ "${LOCAL_TASKS}" = "1" ]; then fdisk -l "/dev/${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${disk}" done cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions + + ## Dump iptables + test -x /sbin/iptables && /sbin/iptables -L -n -v > ${LOCAL_BACKUP_DIR}/iptables.txt && /sbin/iptables -t filter -L -n -v >> ${LOCAL_BACKUP_DIR}/iptables.txt + else ## Dump network connections with netstat netstat -finet -atn > ${LOCAL_BACKUP_DIR}/netstat.out @@ -270,6 +274,10 @@ if [ "${LOCAL_TASKS}" = "1" ]; then ## Dump MBR / table partitions ##disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions + + ## Dump pf infos + pfctl -sa |> ${LOCAL_BACKUP_DIR}/pfctl-sa.txt + fi ## Dump rights From 055ee8dd90f93ed8e4ec37594fdce3db643b8a90 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 26 Sep 2019 09:32:51 +0200 Subject: [PATCH 42/49] kill process if really exists, otherwise remove the pid file --- zzz_evobackup | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index d0225f0..6b48587 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -116,13 +116,18 @@ pick_server() { ## Verify other evobackup process and kill if needed if [ -e "${PIDFILE}" ]; then pid=$(cat "${PIDFILE}") - # Killing the childs of evobackup. - for ppid in $(pgrep -P "${pid}"); do - kill -9 "${ppid}"; - done - # Then kill the main PID. - kill -9 "${pid}" - printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\n" >&2 + # Does process still exist ? + if kill -O ${pid} 2> /dev/null; then + # Killing the childs of evobackup. + for ppid in $(pgrep -P "${pid}"); do + kill -9 "${ppid}"; + done + # Then kill the main PID. + kill -9 "${pid}" + printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\n" >&2 + else + rm -f ${PIDFILE} + fi fi echo "$$" > ${PIDFILE} # shellcheck disable=SC2064 From 4eb5e61ff00fc2fd4c628ce9298cd0cdf714f9a6 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 18 Oct 2019 11:53:29 +0200 Subject: [PATCH 43/49] improve iptables backup readability --- zzz_evobackup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 6b48587..d52db8c 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -268,7 +268,9 @@ if [ "${LOCAL_TASKS}" = "1" ]; then cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions ## Dump iptables - test -x /sbin/iptables && /sbin/iptables -L -n -v > ${LOCAL_BACKUP_DIR}/iptables.txt && /sbin/iptables -t filter -L -n -v >> ${LOCAL_BACKUP_DIR}/iptables.txt + if [ -x /sbin/iptables ]; then + { /sbin/iptables -L -n -v; /sbin/iptables -t filter -L -n -v; } > ${LOCAL_BACKUP_DIR}/iptables.txt + fi else ## Dump network connections with netstat From da40afa99cf0d22f8b985f97060959939f722b33 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 29 Oct 2019 17:30:47 +0100 Subject: [PATCH 44/49] Add findmnt(8) output in the backup --- zzz_evobackup | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zzz_evobackup b/zzz_evobackup index d52db8c..3ebe7e2 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -272,6 +272,11 @@ if [ "${LOCAL_TASKS}" = "1" ]; then { /sbin/iptables -L -n -v; /sbin/iptables -t filter -L -n -v; } > ${LOCAL_BACKUP_DIR}/iptables.txt fi + ## Dump findmnt(8) output + FINDMNT_BIN=$(command -v findmnt) + if [ -x ${FINDMNT_BIN} ]; then + ${FINDMNT_BIN} > ${LOCAL_BACKUP_DIR}/findmnt.txt + fi else ## Dump network connections with netstat netstat -finet -atn > ${LOCAL_BACKUP_DIR}/netstat.out From 832a219f0badc2dbe40896321da4ee8e277da724 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 12 Nov 2019 14:49:16 +0100 Subject: [PATCH 45/49] "kill -O" doesn't exist, "kill -0" does! --- zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 3ebe7e2..d20c6d8 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -117,7 +117,7 @@ pick_server() { if [ -e "${PIDFILE}" ]; then pid=$(cat "${PIDFILE}") # Does process still exist ? - if kill -O ${pid} 2> /dev/null; then + if kill -0 ${pid} 2> /dev/null; then # Killing the childs of evobackup. for ppid in $(pgrep -P "${pid}"); do kill -9 "${ppid}"; From fe6811bc1e450db3a78ba5a622bd041eed6b1144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 15 Dec 2019 18:52:09 +0100 Subject: [PATCH 46/49] check-incs.sh : whitespaces and braces --- check-incs.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/check-incs.sh b/check-incs.sh index 3cd6c36..fb9335e 100644 --- a/check-incs.sh +++ b/check-incs.sh @@ -5,35 +5,35 @@ EVOBACKUP_CONFIGS="/etc/evobackup/*" relative_date() { format=$(echo $1 | cut -d'.' -f1) time_jump=$(echo $1 | cut -d'.' -f2) - + reference_date=$(date "${format}") past_date=$(date --date "${reference_date} ${time_jump}" +"%Y-%m-%d") - + echo ${past_date} } inc_exists() { - ls -d /backup/incs/$1 > /dev/null 2>&1 + ls -d /backup/incs/$1 > /dev/null 2>&1 } jail_exists() { - ls -d /backup/jails/$1 > /dev/null 2>&1 + ls -d /backup/jails/$1 > /dev/null 2>&1 } # default return value is 0 (succes) rc=0 # loop for each configured jail for file in ${EVOBACKUP_CONFIGS}; do - jail_name=$(basename $file) + jail_name=$(basename ${file}) # check if jail is present if jail_exists ${jail_name}; then # get jail last configuration date - jail_config_age=$(date --date "$(stat -c %y ${file})" +%s) + jail_config_age=$(date --date "$(stat -c %y ${file})" +"%s") # loop for each line in jail configuration - for line in $(cat $file); do + for line in $(cat ${file}); do # inc date in ISO format - inc_date=$(relative_date $line) + inc_date=$(relative_date ${line}) # inc date in seconds from epoch - inc_age=$(date --date "${inc_date}" +%s) - # check if the configuration changed after the inc date - if [ $jail_config_age -lt $inc_age ]; then + inc_age=$(date --date "${inc_date}" +"%s") + # check if the configuration changed after the inc date + if [ "${jail_config_age}" -lt "${inc_age}" ]; then # Error if inc is not found if ! inc_exists ${jail_name}/${inc_date}*; then echo "ERROR: inc is missing \`${jail_name}/${inc_date}'" >&2 From 86f6089ce67f40b149a680ab4db163d52ae01a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 15 Dec 2019 18:54:57 +0100 Subject: [PATCH 47/49] check-last-incs.sh : new script to quickly check last inc for jails --- check-last-incs.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 check-last-incs.sh diff --git a/check-last-incs.sh b/check-last-incs.sh new file mode 100644 index 0000000..cefcb89 --- /dev/null +++ b/check-last-incs.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +inc_exists() { + ls -d /backup/incs/$1 > /dev/null 2>&1 +} +# default return value is 0 (succes) +rc=0 +# loop for each found jail +for file in /backup/jails/*; do + jail_name=$(basename ${file}) + # inc date in seconds from epoch + inc_date=$(date --date "yesterday" +"%Y-%m-%d") + # Error if inc is not found + if ! inc_exists ${jail_name}/${inc_date}*; then + echo "ERROR: inc is missing \`${jail_name}/${inc_date}'" >&2 + rc=1 + fi +done + +exit $rc From 23c98f64aaa7884d1d4871ff6a3c6029f3f3dee0 Mon Sep 17 00:00:00 2001 From: Victor LABORIE Date: Wed, 8 Jan 2020 14:31:43 +0100 Subject: [PATCH 48/49] Don't force Vagrant::DEFAULT_SERVER_URL (doesn't work with recent Vagrant version) --- Vagrantfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 8d0b67d..fe5d6ab 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,8 +1,6 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -Vagrant::DEFAULT_SERVER_URL.replace('https://vagrantcloud.com') - # Load ~/.VagrantFile if exist, permit local config provider vagrantfile = File.join("#{Dir.home}", '.VagrantFile') load File.expand_path(vagrantfile) if File.exists?(vagrantfile) From 24ffa76dfcb1b934fd4eec8fb7a4c5332496c791 Mon Sep 17 00:00:00 2001 From: Victor LABORIE Date: Wed, 5 Feb 2020 14:58:15 +0100 Subject: [PATCH 49/49] Do not unnecessary use is-on and reload in ip/port/key (Fix #31) --- lib/bkctld-ip | 2 +- lib/bkctld-key | 2 -- lib/bkctld-port | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/bkctld-ip b/lib/bkctld-ip index 5512b12..32fc246 100755 --- a/lib/bkctld-ip +++ b/lib/bkctld-ip @@ -30,6 +30,6 @@ else done sed -i "s~^AllowUsers .*~${allow}~" "${JAILDIR}/$jail/${SSHD_CONFIG}" notice "${jail} : update ip => ${ip}" - "${LIBDIR}/bkctld-is-on" "${jail}" && "${LIBDIR}/bkctld-reload" "${jail}" + "${LIBDIR}/bkctld-reload" "${jail}" "${LIBDIR}/bkctld-firewall" "${jail}" fi diff --git a/lib/bkctld-key b/lib/bkctld-key index 5fb8c53..5bab190 100755 --- a/lib/bkctld-key +++ b/lib/bkctld-key @@ -22,6 +22,4 @@ else cat "${keyfile}" > "${JAILDIR}/${jail}/${AUTHORIZED_KEYS}" chmod 600 "${JAILDIR}/${jail}/${AUTHORIZED_KEYS}" notice "${jail} : update key => ${keyfile}" - - "${LIBDIR}/bkctld-is-on" "${jail}" && "${LIBDIR}/bkctld-reload" "${jail}" fi diff --git a/lib/bkctld-port b/lib/bkctld-port index 6f86092..b13f300 100755 --- a/lib/bkctld-port +++ b/lib/bkctld-port @@ -23,6 +23,6 @@ else fi sed -i "s/^Port .*/Port ${port}/" "${JAILDIR}/$jail/${SSHD_CONFIG}" notice "${jail} : update port => ${port}" - "${LIBDIR}/bkctld-is-on" "${jail}" && "${LIBDIR}/bkctld-reload" "${jail}" + "${LIBDIR}/bkctld-reload" "${jail}" "${LIBDIR}/bkctld-firewall" "${jail}" fi