From 2aeba60095c2dd341fb74f242f9d4862b5bc1d88 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 19 Dec 2019 14:17:15 +0100 Subject: [PATCH 1/5] better install documentation (cron jobs) --- docs/install.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/install.md b/docs/install.md index 22d3aed..d27adab 100644 --- a/docs/install.md +++ b/docs/install.md @@ -16,16 +16,16 @@ The chroot jails depend on these packages ~~~ apt install \ - bash \ - coreutils \ - sed \ - dash \ - mount \ - rsync \ - openssh-server \ - openssh-sftp-server \ - libc6-i386 \ - libc6 + bash \ + coreutils \ + sed \ + dash \ + mount \ + rsync \ + openssh-server \ + openssh-sftp-server \ + libc6-i386 \ + libc6 ~~~ ## Client dependencies @@ -37,11 +37,13 @@ The clients only require OpenSSH and rsync. Edit the root crontab ~~~ -# crontab -e -+ 30 10 * * * /usr/sbin/bkctld inc && /usr/sbin/bkctld rm +# $editor /etc/cron.d/bkctld ++ MAILTO=alert4@evolix.net ++ 30 11 * * * root /usr/sbin/bkctld inc && /usr/sbin/bkctld rm ++ 30 23 * * * root /usr/share/scripts/check-incs.sh 1> /dev/null ~~~ ## Notes If you want mutiples backups in a day (1 by hour maximum) you can run `bkctld inc` multiples times, if you want to keep incremental -backups **for ever**, just don't run `bkctld rm`. \ No newline at end of file +backups **for ever**, just don't run `bkctld rm`. From 48ca7cc8e421d248865a650fcf7e3cdcfe2c622b Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 13 Feb 2020 14:34:09 +0100 Subject: [PATCH 2/5] add a schema-only dump for MySQL databases --- zzz_evobackup | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index d20c6d8..28d4e40 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -154,7 +154,14 @@ if [ "${LOCAL_TASKS}" = "1" ]; then # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \ # --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i; done - ## example with compressed SQL dump for each databases + ## example with SQL dump (schema only, no data) for each databases + # mkdir -p -m 700 /home/mysqldump/ + # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \ + # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do + # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases $i > /home/mysqldump/${i}.schema.sql + # done + + ## example with compressed SQL dump (with data) for each databases # mkdir -p -m 700 /home/mysqldump/ # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \ # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do From 124af075ae0ecccf39d2eeb0a8b3463be660166d Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 13 Feb 2020 14:34:49 +0100 Subject: [PATCH 3/5] improve partitions dump --- zzz_evobackup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 28d4e40..1faaf36 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -267,10 +267,10 @@ if [ "${LOCAL_TASKS}" = "1" ]; then apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available ## Dump MBR / table partitions - disks=$(lsblk -l | grep disk | grep -v drbd | awk '{print $1}') + disks=$(lsblk -l | grep disk | grep -v -E '(drbd|fd[0-9]+)' | awk '{print $1}') for disk in ${disks}; do dd if="/dev/${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)" - fdisk -l "/dev/${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${disk}" + fdisk -l "/dev/${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${disk}" 2>&1 done cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions From 6d7072234df0394870634e5561c50fe81126bfd1 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 14 Feb 2020 10:52:54 +0100 Subject: [PATCH 4/5] improve comments readability --- zzz_evobackup | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 1faaf36..03692a0 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -141,7 +141,7 @@ if [ "${LOCAL_TASKS}" = "1" ]; then ## OpenLDAP : example with slapcat # slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak - ### MySQL + ## MySQL ## example with global and compressed mysqldump # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ @@ -188,7 +188,7 @@ if [ "${LOCAL_TASKS}" = "1" ]; then # fi # done - ### PostgreSQL + ## PostgreSQL ## example with pg_dumpall (warning: you need space in ~postgres) # su - postgres -c "pg_dumpall > ~/pg.dump.bak" @@ -204,7 +204,8 @@ if [ "${LOCAL_TASKS}" = "1" ]; then ## example with only TABLE1 and TABLE2 from MYBASE # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE - ## MongoDB : example with mongodump + ## MongoDB + ## don't forget to create use with read-only access ## > use admin ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } ) @@ -215,10 +216,14 @@ if [ "${LOCAL_TASKS}" = "1" ]; then # echo "Error with mongodump!" # fi - ## Redis : example with copy .rdb file + ## Redis + + ## example with copy .rdb file # cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/ - ## ElasticSearch, take a snapshot as a backup. + ## ElasticSearch + + ## Take a snapshot as a backup. ## Warning: You need to have a path.repo configured. ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log @@ -239,10 +244,13 @@ if [ "${LOCAL_TASKS}" = "1" ]; then # date=$(date +%F) # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log - ## RabbitMQ : export config + ## RabbitMQ + + ## export config #rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> $LOGFILE - # backup MegaCli config + ## MegaCli config + #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null ## Dump system and kernel versions @@ -295,7 +303,7 @@ if [ "${LOCAL_TASKS}" = "1" ]; then ##disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions ## Dump pf infos - pfctl -sa |> ${LOCAL_BACKUP_DIR}/pfctl-sa.txt + pfctl -sa > ${LOCAL_BACKUP_DIR}/pfctl-sa.txt fi From ed8340480672100ae0a4dc7f4dfbdbbf272f79de Mon Sep 17 00:00:00 2001 From: Victor LABORIE Date: Thu, 5 Mar 2020 14:42:29 +0100 Subject: [PATCH 5/5] Add lock for bkctld inc / rm when not using btrfs (Fix #32) --- lib/bkctld-inc | 17 ++++++++++++++--- lib/bkctld-rm | 36 ++++++++++++++++-------------------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/lib/bkctld-inc b/lib/bkctld-inc index 3ee7fb8..2f359ee 100755 --- a/lib/bkctld-inc +++ b/lib/bkctld-inc @@ -15,11 +15,22 @@ for jail in $("${LIBDIR}/bkctld-list"); do jail_inode=$(stat --format=%i "${JAILDIR}/${jail}") if [ "$jail_inode" -eq 256 ]; then /bin/btrfs subvolume snapshot -r "${JAILDIR}/${jail}" "${inc}" | debug + end=$(date +"%H:%M:%S") + notice "${jail} : made ${date} inc [${start}/${end}]" else - cp -alx "${JAILDIR}/${jail}/" "${inc}" | debug + lock="/run/lock/bkctld/inc-${jail}.lock" + if [ -f "${lock}" ]; then + warning "${jail} : trying to run already running inc" + else + ( + mkdir -p /run/lock/bkctld && touch "${lock}" + trap "rm -f ${lock}" 0 + cp -alx "${JAILDIR}/${jail}/" "${inc}" + end=$(date +"%H:%M:%S") + notice "${jail} : made ${date} inc [${start}/${end}]" + ) & + fi fi - end=$(date +"%H:%M:%S") - notice "${jail} : made ${date} inc [${start}/${end}]" else warning "${jail} : trying to made already existant inc" fi diff --git a/lib/bkctld-rm b/lib/bkctld-rm index 1e79a36..1139b65 100755 --- a/lib/bkctld-rm +++ b/lib/bkctld-rm @@ -6,19 +6,6 @@ LIBDIR="$(dirname $0)" && . "${LIBDIR}/config" -empty="/tmp/bkctld-${$}-$(date +%N))" -mkdir "${empty}" -pidfile="/var/run/bkctld-rm.pid" -if [ -f "${pidfile}" ]; then - pid=$(cat "${pidfile}") - ps -u "${pid}" >/dev/null - if [ "${?}" -eq 0 ]; then - kill -9 "${pid}" - warning "${0} rm always run (PID ${pid}), killed by ${$} !" - fi - rm "${pidfile}" - fi -echo "${$}" > "${pidfile}" for jail in $("${LIBDIR}/bkctld-list"); do incs=$(ls "${INCDIR}/${jail}") if [ -f "${CONFDIR}/${jail}" ]; then @@ -33,16 +20,25 @@ for jail in $("${LIBDIR}/bkctld-list"); do inc_inode=$(stat --format=%i "${INCDIR}/${jail}/${j}") if [ "${inc_inode}" -eq 256 ]; then /bin/btrfs subvolume delete "${INCDIR}/${jail}/${j}" | debug + end=$(date +"%H:%M:%S") + notice "${jail} : deleted ${j} inc [${start}/${end}]" else - cd "${INCDIR}/${jail}" - rsync -a --delete "${empty}/" "${j}/" - rmdir "${j}" + lock="/run/lock/bkctld/rm-${jail}.lock" + if [ -f "${lock}" ]; then + warning "${jail} : trying to run already running rm" + else + ( + empty="/tmp/bkctld-${$}-$(date +%N)" + mkdir -p /run/lock/bkctld && touch "${lock}" && mkdir -p "${empty}" + trap "rm -f ${lock} && rmdir ${empty}" 0 + rsync -a --delete "${empty}/" "${INCDIR}/${jail}/${j}/" + rmdir "${j}" + end=$(date +"%H:%M:%S") + notice "${jail} : deleted ${j} inc [${start}/${end}]" + ) & + fi fi - end=$(date +"%H:%M:%S") - notice "${jail} : deleted ${j} inc [${start}/${end}]" done rm "${keepfile}" fi done -rmdir "${empty}" -rm "${pidfile}"