Compare commits

..

2 commits

Author SHA1 Message Date
David Prevot 13c57aecab New upstream version 22.06
Some checks are pending
continuous-integration/drone/push Build is passing
gitea.evolix.org on plain agent00/evobackup/pipeline/head This commit looks good
gitea.evolix.org/evobackup/pipeline/head Build started...
gitea/evobackup/pipeline/head This commit looks good
2022-06-28 16:34:02 +02:00
David Prevot 0980100739 New upstream version 22.04 2022-05-03 16:40:07 +02:00
18 changed files with 279 additions and 424 deletions

View file

@ -1,15 +1,24 @@
pipeline {
agent { label 'sbuild' }
agent { label 'docker' }
stages {
stage('Build Debian package') {
agent {
docker {
image 'evolix/gbp:bullseye'
args '-u root --privileged -v /tmp:/tmp'
}
}
when {
branch 'debian'
}
steps {
script {
sh 'gbp buildpackage'
sh 'mk-build-deps --install --remove debian/control'
sh 'rm -rf source'
sh "gbp clone --debian-branch=$GIT_BRANCH $GIT_URL source"
sh 'cd source && git checkout $GIT_BRANCH && gbp buildpackage -us -uc'
}
archiveArtifacts allowEmptyArchive: true, artifacts: 'build-area/*.gz,build-area/*.bz2,build-area/*.xz,build-area/*.deb,build-area/*.dsc,build-area/*.changes,build-area/*.buildinfo,build-area/*.build,build-area/lintian.txt'
archiveArtifacts allowEmptyArchive: true, artifacts: '*.gz,*.bz2,*.xz,*.deb,*.dsc,*.changes,*.buildinfo,lintian.txt'
}
}
@ -19,7 +28,10 @@ pipeline {
}
steps {
script {
sh 'rsync -avP build-area/bkctld*.deb build-area/bkctld*.changes build-area/bkctld*.buildinfo pub.evolix.org:/srv/upload/'
sh 'echo Dummy line to remove once something actually happens.'
/* No crendentials yet
sh 'rsync -avP /tmp/bkctld/ droneci@pub.evolix.net:/home/droneci/bkctld/'
*/
}
}
}

46
.drone.yml Normal file
View file

@ -0,0 +1,46 @@
kind: pipeline
name: default
steps:
- name: fetch
image: alpine/git
commands:
- git fetch --tags
- name: build debian package
image: evolix/gbp:bullseye
branches:
- debian
commands:
- mk-build-deps --install --remove debian/control
- git clean --force
- gbp buildpackage -us -uc
volumes:
- name: tmp
path: /tmp
when:
branch:
- debian
- name: upload debian package
image: drillster/drone-rsync
settings:
hosts: ["pub.evolix.net"]
port: 22
user: droneci
key:
from_secret: drone_private_key
target: /home/droneci/bkctld/
source: /tmp/bkctld/
delete: true
volumes:
- name: tmp
path: /tmp
when:
branch:
- debian
volumes:
- name: tmp
host:
path: /tmp

2
.gitignore vendored
View file

@ -1,2 +0,0 @@
*.swp
.vagrant

View file

@ -10,17 +10,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
### Deprecated
### Removed
### Fixed
### Security
## [22.12]
### Changed
* Use --dump-dir instead of --backup-dir to suppress dump-server-state warning
* Do not use rsync compression
* Replace rsync option --verbose by --itemize-changes
@ -28,20 +17,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* update-evobackup-canary: do not use GNU date, for it to be compatible with OpenBSD
* Add AGPL License and README
* Script now depends on Bash
* tolerate absence of mtr or traceroute
* Only one loop for all Redis instances
* remodel how we build the rsync command
* use sub shells instead of moving around
* Separate Rsync for the canary file if the main Rsync has finished without errors
### Deprecated
### Removed
* No more fallback if dump-server-state is missing
### Fixed
* Make start_time and stop_time compatible with OpenBSD
### Security
## [22.03]
Split client and server parts of the project

View file

@ -1,3 +1,3 @@
Pour l'installation de `zzz_evobackup`, voir <https://intra.evolix.net/OutilsInternes/EvoBackupClient#installer-et-configurer-le-client-evobackup>
Pour l'installation de `zzz_evobackup`, voir <https://intra.evolix.net/Installation_jail_backup_Evolix#installation-du-client-evobackup>
Pour `update-evobackup-canary`, voir <https://intra.evolix.net/OutilsInternes/update-evobackup-canary>
Pour `update-evobackup-canary`, voir <https://intra.evolix.net/OutilsInternes/update-evobackup-canary>

View file

@ -18,7 +18,7 @@
##### Configuration ###################################################
VERSION="22.12"
VERSION="22.05"
# email adress for notifications
MAIL=jdoe@example.com
@ -48,9 +48,6 @@ PIDFILE="/var/run/${PROGNAME}.pid"
# Customize the log path if you have multiple scripts and with separate logs
LOGFILE="/var/log/evobackup.log"
# Full Rsync log file, reset each time
RSYNC_LOGFILE="/var/log/${PROGNAME}.rsync.log"
HOSTNAME=$(hostname)
DATE_FORMAT="%Y-%m-%d %H:%M:%S"
@ -60,77 +57,7 @@ DATE_FORMAT="%Y-%m-%d %H:%M:%S"
# Enable/disable sync tasks (default: enabled)
: "${SYNC_TASKS:=1}"
CANARY_FILE="/zzz_evobackup_canary"
# Source paths can be customized
# Empty lines, and lines containing # or ; are ignored
# NOTE: remember to single-quote paths if they contain globs (*)
# and you want to defer expansion
RSYNC_INCLUDES="
/etc
/root
/var
/home
"
# Excluded paths can be customized
# Empty lines, and lines beginning with # or ; are ignored
# NOTE: remember to single-quote paths if they contain globs (*)
# and you want to defer expansion
RSYNC_EXCLUDES="
/dev
/proc
/run
/sys
/tmp
/usr/doc
/usr/obj
/usr/share/doc
/usr/src
/var/apt
/var/cache
'/var/db/munin/*.tmp'
/var/lib/amavis/amavisd.sock
/var/lib/amavis/tmp
/var/lib/amavis/virusmails
'/var/lib/clamav/*.tmp'
/var/lib/elasticsearch
/var/lib/metche
/var/lib/mongodb
'/var/lib/munin/*tmp*'
/var/lib/mysql
/var/lib/php/sessions
/var/lib/php5
/var/lib/postgres
/var/lib/postgresql
/var/lib/sympa
/var/lock
/var/run
/var/spool/postfix
/var/spool/smtpd
/var/spool/squid
/var/state
/var/tmp
lost+found
'.nfs.*'
'lxc/*/rootfs/tmp'
'lxc/*/rootfs/usr/doc'
'lxc/*/rootfs/usr/obj'
'lxc/*/rootfs/usr/share/doc'
'lxc/*/rootfs/usr/src'
'lxc/*/rootfs/var/apt'
'lxc/*/rootfs/var/cache'
'lxc/*/rootfs/var/lib/php5'
'lxc/*/rootfs/var/lib/php/sessions'
'lxc/*/rootfs/var/lock'
'lxc/*/rootfs/var/run'
'lxc/*/rootfs/var/state'
'lxc/*/rootfs/var/tmp'
/home/mysqltmp
"
##### FUNCTIONS #######################################################
##### SETUP AND FUNCTIONS #############################################
local_tasks() {
log "START LOCAL_TASKS"
@ -149,7 +76,7 @@ local_tasks() {
# rm -rf ${LOCAL_BACKUP_DIR}/mysql
# rm -rf ${LOCAL_BACKUP_DIR}/mysqlhotcopy
# rm -rf /home/mysqldump
# find ${LOCAL_BACKUP_DIR}/ -type f -name '*.err' -delete
# rm -f ${LOCAL_BACKUP_DIR}/*.err ${LOCAL_BACKUP_DIR}/**/*.err
## example with global and compressed mysqldump
# mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
@ -163,7 +90,7 @@ local_tasks() {
## example with compressed SQL dump (with data) for each databases
# mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
# for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
# | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)"); do
# | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
# mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i 2> ${LOCAL_BACKUP_DIR}/${i}.err | gzip --best > ${LOCAL_BACKUP_DIR}/mysql/${i}.sql.gz
# last_rc=$?
# if [ ${last_rc} -ne 0 ]; then
@ -192,7 +119,7 @@ local_tasks() {
## example with SQL dump (schema only, no data) for each databases
# mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
# for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
# | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)"); do
# | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
# mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases $i 2> ${LOCAL_BACKUP_DIR}/${i}.schema.err > ${LOCAL_BACKUP_DIR}/mysql/${i}.schema.sql
# last_rc=$?
# if [ ${last_rc} -ne 0 ]; then
@ -214,7 +141,7 @@ local_tasks() {
## example with two dumps for each table (.sql/.txt) for all databases
# for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
# | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)" ); do
# | egrep -v "^(Database|information_schema|performance_schema|sys)" ); do
# mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump
# mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \
# --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i 2> /home/mysqldump/$i.err"
@ -236,7 +163,7 @@ local_tasks() {
## example for multiples MySQL instances
# mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3)
# grep --extended-regexp "^port\s*=\s*\d*" /etc/mysql/my.cnf | while read instance; do
# grep -E "^port\s*=\s*\d*" /etc/mysql/my.cnf |while read instance; do
# instance=$(echo "$instance"|awk '{ print $3 }')
# if [ "$instance" != "3306" ]
# then
@ -255,16 +182,13 @@ local_tasks() {
# rm -rf ${LOCAL_BACKUP_DIR}/pg.*.gz
# rm -rf ${LOCAL_BACKUP_DIR}/pg-backup.tar
# rm -rf ${LOCAL_BACKUP_DIR}/postgresql/*
## example with pg_dumpall (warning: you need space in ~postgres)
# su - postgres -c "pg_dumpall > ~/pg.dump.bak"
# mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/
## another method with gzip directly piped
# (
# cd /var/lib/postgresql;
# sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
# )
# cd /var/lib/postgresql
# sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
# cd - > /dev/null
## example with all tables from MYBASE excepts TABLE1 and TABLE2
# pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE
@ -275,11 +199,9 @@ local_tasks() {
## example with compressed PostgreSQL dump for each databases
# mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/postgresql
# chown postgres:postgres ${LOCAL_BACKUP_DIR}/postgresql
# (
# cd /var/lib/postgresql
# dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*)
# for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done
# )
# dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*)
#
# for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done
## MongoDB
@ -299,14 +221,15 @@ local_tasks() {
## Purge previous dumps
# rm -rf ${LOCAL_BACKUP_DIR}/redis/
# rm -rf ${LOCAL_BACKUP_DIR}/redis-*
## Copy dump.rdb file for each found instance
# for instance in $(find /var/lib/ -mindepth 1 -maxdepth 1 '(' -type d -o -type l ')' -name 'redis*'); do
# if [ -f "${instance}/dump.rdb" ]; then
# name=$(basename $instance)
# mkdir -p ${LOCAL_BACKUP_DIR}/${name}
# cp -a "${instance}/dump.rdb" "${LOCAL_BACKUP_DIR}/${name}"
# gzip "${LOCAL_BACKUP_DIR}/${name}/dump.rdb"
# fi
## example with copy .rdb file
## for the default instance :
# mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/redis/
# cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/redis/
## for multiple instances :
# for instance in $(ls -d /var/lib/redis-*); do
# name=$(basename $instance)
# mkdir -p ${LOCAL_BACKUP_DIR}/${name}
# cp -a ${instance}/dump.rdb ${LOCAL_BACKUP_DIR}/${name}
# done
## ElasticSearch
@ -325,7 +248,7 @@ local_tasks() {
# else
# echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
# fi
## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
# for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
# done
@ -335,35 +258,24 @@ local_tasks() {
## RabbitMQ
## export config
# rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> "${LOGFILE}"
#rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> "${LOGFILE}"
## MegaCli config
# megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null
#megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null
## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls)
network_targets="8.8.8.8 www.evolix.fr travaux.evolix.net"
mtr_bin=$(command -v mtr)
if [ -n "${mtr_bin}" ]; then
for addr in ${network_targets}; do
${mtr_bin} -r "${addr}" > "${LOCAL_BACKUP_DIR}/mtr-${addr}"
done
fi
traceroute_bin=$(command -v traceroute)
if [ -n "${traceroute_bin}" ]; then
for addr in ${network_targets}; do
${traceroute_bin} -n "${addr}" > "${LOCAL_BACKUP_DIR}/traceroute-${addr}" 2>&1
done
fi
for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do
mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr}
traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1
done
server_state_dir="${LOCAL_BACKUP_DIR}/server-state"
dump_server_state_bin=$(command -v dump-server-state)
if [ -z "${dump_server_state_bin}" ]; then
error "dump-server-state is missing"
rc=1
else
if [ "${SYSTEM}" = "linux" ]; then
if [ "${SYSTEM}" = "linux" ]; then
if [ -n "${dump_server_state_bin}" ]; then
${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
last_rc=$?
if [ ${last_rc} -ne 0 ]; then
@ -371,93 +283,81 @@ local_tasks() {
rc=1
fi
else
mkdir -p "${server_state_dir}"
## Dump system and kernel versions
uname -a > ${server_state_dir}/uname.txt
## Dump process with ps
ps auwwx > ${server_state_dir}/ps.txt
## Dump network connections with ss
ss -taupen > ${server_state_dir}/netstat.txt
## List Debian packages
dpkg -l > ${server_state_dir}/packages
dpkg --get-selections > ${server_state_dir}/packages.getselections
apt-cache dumpavail > ${server_state_dir}/packages.available
## Dump iptables
if [ -x /sbin/iptables ]; then
{ /sbin/iptables -L -n -v; /sbin/iptables -t filter -L -n -v; } > ${server_state_dir}/iptables.txt
fi
## Dump findmnt(8) output
FINDMNT_BIN=$(command -v findmnt)
if [ -x "${FINDMNT_BIN}" ]; then
${FINDMNT_BIN} > ${server_state_dir}/findmnt.txt
fi
## Dump MBR / table partitions
disks=$(lsblk -l | grep disk | grep -v -E '(drbd|fd[0-9]+)' | awk '{print $1}')
for disk in ${disks}; do
dd if="/dev/${disk}" of="${server_state_dir}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)"
fdisk -l "/dev/${disk}" > "${server_state_dir}/partitions-${disk}" 2>&1
done
cat ${server_state_dir}/partitions-* > ${server_state_dir}/partitions
fi
else
if [ -n "${dump_server_state_bin}" ]; then
${dump_server_state_bin} --all --force --dump-dir "${server_state_dir}"
last_rc=$?
if [ ${last_rc} -ne 0 ]; then
error "dump-server-state returned an error ${last_rc}, check ${server_state_dir}"
rc=1
fi
else
mkdir -p "${server_state_dir}"
## Dump system and kernel versions
uname -a > ${server_state_dir}/uname
## Dump process with ps
ps auwwx > ${server_state_dir}/ps.out
## Dump network connections with fstat
fstat | head -1 > ${server_state_dir}/netstat.out
fstat | grep internet >> ${server_state_dir}/netstat.out
## List OpenBSD packages
pkg_info -m > ${server_state_dir}/packages
## Dump MBR / table partitions
disklabel sd0 > ${server_state_dir}/partitions
## Dump pf infos
pfctl -sa > ${server_state_dir}/pfctl-sa.txt
fi
fi
## Dump rights
# getfacl -R /var > ${server_state_dir}/rights-var.txt
# getfacl -R /etc > ${server_state_dir}/rights-etc.txt
# getfacl -R /usr > ${server_state_dir}/rights-usr.txt
# getfacl -R /home > ${server_state_dir}/rights-home.txt
#getfacl -R /var > ${server_state_dir}/rights-var.txt
#getfacl -R /etc > ${server_state_dir}/rights-etc.txt
#getfacl -R /usr > ${server_state_dir}/rights-usr.txt
#getfacl -R /home > ${server_state_dir}/rights-home.txt
log "STOP LOCAL_TASKS"
}
build_rsync_main_cmd() {
###################################################################
# /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ #
###################################################################
# DO NOT USE COMMENTS in rsync lines #
# DO NOT ADD WHITESPACES AFTER \ in rsync lines #
# It breaks the command and destroys data #
# You should not modify this, unless you are really REALLY sure #
###################################################################
# Create a temp file for excludes and includes
includes_file="$(mktemp --tmpdir "${PROGNAME}.includes.XXXXXX")"
excludes_file="$(mktemp --tmpdir "${PROGNAME}.excludes.XXXXXX")"
# … and add them to the list of files to delete at exit
temp_files="${includes_file} ${excludes_file}"
trap "rm -f ${temp_files}" EXIT
# Store includes/excludes in files
# without blank lines of comments (# or ;)
echo "${RSYNC_INCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${includes_file}"
echo "${RSYNC_EXCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${excludes_file}"
# Rsync command
cmd="$(command -v rsync)"
# Rsync main options
cmd="${cmd} --archive"
cmd="${cmd} --itemize-changes"
cmd="${cmd} --quiet"
cmd="${cmd} --stats"
cmd="${cmd} --human-readable"
cmd="${cmd} --relative"
cmd="${cmd} --partial"
cmd="${cmd} --delete"
cmd="${cmd} --delete-excluded"
cmd="${cmd} --force"
cmd="${cmd} --ignore-errors"
cmd="${cmd} --log-file=${RSYNC_LOGFILE}"
cmd="${cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'"
# Rsync excludes
while read line ; do
cmd="${cmd} --exclude ${line}"
done < "${excludes_file}"
# Rsync local sources
cmd="${cmd} ${default_includes}"
while read line ; do
cmd="${cmd} ${line}"
done < "${includes_file}"
# Rsync remote destination
cmd="${cmd} root@${SSH_SERVER}:/var/backup/"
# output final command
echo "${cmd}"
}
build_rsync_canary_cmd() {
# Rsync command
cmd="$(command -v rsync)"
# Rsync options
cmd="${cmd} --rsh='ssh -p ${SSH_PORT} -o \"ConnectTimeout ${SSH_CONNECT_TIMEOUT}\"'"
# Rsync local source
cmd="${cmd} ${CANARY_FILE}"
# Rsync remote destination
cmd="${cmd} root@${SSH_SERVER}:/var/backup/"
# output final command
echo "${cmd}"
}
sync_tasks() {
n=0
server=""
@ -482,48 +382,86 @@ sync_tasks() {
SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
SSH_PORT=$(echo "${server}" | cut -d':' -f2)
if [ "${SYSTEM}" = "linux" ]; then
rep="/bin /boot /lib /opt /sbin /usr"
else
rep="/bsd /bin /sbin /usr"
fi
log "START SYNC_TASKS - server=${server}"
# default paths, depending on system
if [ "${SYSTEM}" = "linux" ]; then
default_includes="/bin /boot /lib /opt /sbin /usr"
else
default_includes="/bsd /bin /sbin /usr"
fi
update-evobackup-canary --who "${PROGNAME}"
# reset Rsync log file
if [ -n "$(command -v truncate)" ]; then
truncate -s 0 "${RSYNC_LOGFILE}"
else
printf "" > "${RSYNC_LOGFILE}"
fi
# /!\ DO NOT USE COMMENTS in the rsync command /!\
# It breaks the command and destroys data, simply remove (or add) lines.
# Build the final Rsync command
rsync_main_cmd=$(build_rsync_main_cmd)
# Remote shell command
RSH_COMMAND="ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'"
# … log it
log "SYNC_TASKS - Rsync main command : ${rsync_main_cmd}"
# ignore check because we want it to split the different arguments to $rep
# shellcheck disable=SC2086
rsync --archive \
--itemize-changes --stats --human-readable \
--relative --partial \
--delete --delete-excluded --force --ignore-errors \
--exclude "dev" \
--exclude "lost+found" \
--exclude ".nfs.*" \
--exclude "/usr/doc" \
--exclude "/usr/obj" \
--exclude "/usr/share/doc" \
--exclude "/usr/src" \
--exclude "/var/apt" \
--exclude "/var/cache" \
--exclude "/var/lib/amavis/amavisd.sock" \
--exclude "/var/lib/amavis/tmp" \
--exclude "/var/lib/clamav/*.tmp" \
--exclude "/var/lib/elasticsearch" \
--exclude "/var/lib/metche" \
--exclude "/var/lib/munin/*tmp*" \
--exclude "/var/db/munin/*.tmp" \
--exclude "/var/lib/mysql" \
--exclude "/var/lib/php5" \
--exclude "/var/lib/php/sessions" \
--exclude "/var/lib/postgres" \
--exclude "/var/lib/postgresql" \
--exclude "/var/lib/sympa" \
--exclude "/var/lock" \
--exclude "/var/run" \
--exclude "/var/spool/postfix" \
--exclude "/var/spool/smtpd" \
--exclude "/var/spool/squid" \
--exclude "/var/state" \
--exclude "/var/tmp" \
--exclude "lxc/*/rootfs/tmp" \
--exclude "lxc/*/rootfs/usr/doc" \
--exclude "lxc/*/rootfs/usr/obj" \
--exclude "lxc/*/rootfs/usr/share/doc" \
--exclude "lxc/*/rootfs/usr/src" \
--exclude "lxc/*/rootfs/var/apt" \
--exclude "lxc/*/rootfs/var/cache" \
--exclude "lxc/*/rootfs/var/lib/php5" \
--exclude "lxc/*/rootfs/var/lib/php/sessions" \
--exclude "lxc/*/rootfs/var/lock" \
--exclude "lxc/*/rootfs/var/log" \
--exclude "lxc/*/rootfs/var/run" \
--exclude "lxc/*/rootfs/var/state" \
--exclude "lxc/*/rootfs/var/tmp" \
--exclude "/home/mysqltmp" \
${rep} \
/etc \
/root \
/var \
/home \
/zzz_evobackup_canary \
-e "${RSH_COMMAND}" \
"root@${SSH_SERVER}:/var/backup/" \
| tail -30 >> "${LOGFILE}"
# … execute it
eval "${rsync_main_cmd}"
rsync_main_rc=$?
# Copy last lines of rsync log to the main log
tail -n 30 "${RSYNC_LOGFILE}" >> "${LOGFILE}"
if [ ${rsync_main_rc} -ne 0 ]; then
error "rsync returned an error ${rsync_main_rc}, check ${LOGFILE}"
rsync_rc=$?
if [ ${rsync_rc} -ne 0 ]; then
error "rsync returned an error ${rsync_rc}, check ${LOGFILE}"
rc=201
else
# Build the canary Rsync command
rsync_canary_cmd=$(build_rsync_canary_cmd)
# … log it
log "SYNC_TASKS - Rsync canary command : ${rsync_canary_cmd}"
# … execute it
eval "${rsync_canary_cmd}"
fi
log "STOP SYNC_TASKS - server=${server}"
@ -627,16 +565,8 @@ main() {
fi
fi
echo "$$" > "${PIDFILE}"
# Initialize a list of files to delete at exit
# Any file added to the list will also be deleted at exit
temp_files="${PIDFILE}"
# shellcheck disable=SC2064
trap "rm -f ${temp_files}" EXIT
# Update canary to keep track of each run
update-evobackup-canary --who "${PROGNAME}"
trap "rm -f ${PIDFILE}" EXIT
if [ "${LOCAL_TASKS}" = "1" ]; then
local_tasks
@ -667,6 +597,8 @@ export LC_ALL=C
# Error on unassigned variable
set -u
# Fail if a pipeline member returns an error (cf. https://sipb.mit.edu/doc/safe-shell/)
set -o pipefail
# Default return-code (0 == succes)
rc=0

View file

@ -18,24 +18,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Security
## [22.11] - 2022-11-28
### Added
* check-canary: new subcommand to check canary files and content
### Changed
* stats: filter active jails and columnize the output
## [22.07] - 2022-07-20
### Changed
* check-setup: check minifirewall version only if minifirewall is present
* check-setup: get minifirewall version from internal variable (there is no other backward compatible way)
* check-setup: use findmnt with mountpoint instead of target
## [22.06] - 2022-06-28
### Added

View file

@ -4,7 +4,7 @@ Bkctld (aka server-side evobackup)
bkctld helps you manage the receiving side of a backup infrastructure.
It is licensed under the AGPLv3.
With bkctld you create and manage "jails". They contain a chrooted and dedicated SSH server, with its own TCP port and optionally its own set of iptables rules.
With bkctld you create and manage "jails". They contain a chrooted and dedicated SSH server, with it's own TCP port and optionnaly it's own set of iptables rules.
With bkctld you can have hundreds of jails, one for each client to push its data (using Rsync/SFTP). Each client can only see its own data.
@ -30,7 +30,9 @@ This volume can also be encrypted with **LUKS**.
## Security considerations
The client obviously has access to its uploaded data (in the chroot), but the timestamped copies are outside the chroot, to reduce the risk of complete backup erasure from a compromised client.
The client obviously has access to its uploaded data (in the chroot), but the timestamped copies are outside the chroot, to reduce the risk or complete backup erasure from a compromised client.
Since the client connects to the backup server with root, it can mess with the jail and destroy the data. But the timestamped copies are out of reach because outside of the chroot.
It means that **if the client server is compromised**, an attacker can destroy the latest copy of the backed up data, but not the timestamped copies.
And **if the backup server is compromised** an attacker has complete access to all the backup data (inside and outside the jails), but they don't have any access to the client.
@ -39,7 +41,7 @@ This architecture is as secure as SSH, Rsync, chroot and iptables are.
## Install
See the [installation guide](https://intra.evolix.net/OutilsInternes/bkctld) for instructions.
See the [installation guide](docs/install.md) for instructions.
## Testing
@ -49,12 +51,6 @@ You can deploy test environments with Vagrant :
vagrant up
~~~
To destroy Vagrant VMs :
~~~
vagrant destroy
~~~
### Deployment
Run `vagrant rsync-auto` in a terminal for automatic synchronization of
@ -81,8 +77,6 @@ vagrant@buster-btrfs $ sudo -i
root@buster-btrfs # bats /vagrant/test/*.bats
~~~
[comment]: <> (* pour vim)
You should shellcheck your bats files, but with shellcheck > 0.4.6, because the 0.4.0 version doesn't support bats syntax.
## Usage
@ -105,7 +99,7 @@ pandoc -f markdown \
#### Client configuration
You can backup various systems in the evobackup jails : Linux, BSD,
Windows, macOS. The only need is Rsync or an SFTP client.
Windows, macOS. The only need Rsync or an SFTP client.
~~~
rsync -av -e "ssh -p SSH_PORT" /home/ root@SERVER_NAME:/var/backup/home/

View file

@ -53,9 +53,6 @@ while :; do
-f|--force)
export FORCE=1
;;
--no-header)
export HEADER=0
;;
*)
# Default case: If no more options then break out of the loop.
break
@ -67,7 +64,7 @@ done
subcommand="${1:-}"
case "${subcommand}" in
"inc" | "rm" | "check-jails" | "check-setup" | "check-canary" | "stats" | "list")
"inc" | "rm" | "check-jails" | "check-setup" | "stats" | "list")
"${LIBDIR}/bkctld-${subcommand}"
;;
"check")
@ -119,9 +116,7 @@ case "${subcommand}" in
;;
"status")
jail_name="${2:-}"
if [ "${HEADER}" = "1" ]; then
printf '%-30s %-10s %-10s %-25s %-20s\n' 'JAIL NAME' 'STATUS' 'PORT' 'RETENTION (DAY/MONTH)' 'IP'
fi
printf '%-30s %-10s %-10s %-25s %-20s\n' 'JAIL NAME' 'STATUS' 'PORT' 'RETENTION (DAY/MONTH)' 'IP'
if [ "${jail_name}" = "all" ] || [ -z "${jail_name}" ]; then
for jail in $("${LIBDIR}/bkctld-list"); do
"${LIBDIR}/bkctld-${subcommand}" "${jail}"

View file

@ -15,5 +15,3 @@
#LOGLEVEL=6
#NODE=''
#ARCHIVESDIR='/backup/archives'
#WARNING=48
#CRITICAL=72

View file

@ -2,7 +2,15 @@
## Install from package
The install documentation is [here](https://intra.evolix.net/OutilsInternes/bkctld)
A Debian package is available in the Evolix repository
~~~
echo "deb http://pub.evolix.net/ stretch" >> /etc/apt/sources.list
apt update
apt install bkctld
~~~
Then edit `/etc/default/bkctld`
## Instal from sources
@ -11,17 +19,17 @@ Warning: `cp`-ing the files without `-n` or `-i` will replace existing files !
~~~
# git clone https://gitea.evolix.org/evolix/evobackup.git
# cd evobackup
# cp server/bkctld /usr/local/sbin/
# cp bkctld /usr/local/sbin/
# mkdir -p /usr/local/lib/bkctld
# cp server/lib/* /usr/local/lib/bkctld/
# cp lib/* /usr/local/lib/bkctld/
# mkdir -p /usr/local/share/bkctld
# cp server/tpl/* /usr/local/share/bkctld/
# cp server/bkctld.service /lib/systemd/system/
# cp tpl/* /usr/local/share/bkctld/
# cp bkctld.service /lib/systemd/system/
# mkdir -p /usr/local/share/doc/bkctld
# cp client/zzz_evobackup /usr/local/share/doc/bkctld/
# cp zzz_evobackup /usr/local/share/doc/bkctld/
# mkdir -p /usr/local/share/bash_completion/
# cp server/bash_completion /usr/local/share/bash_completion/bkctld
# cp server/bkctld.conf /etc/default/bkctld
# cp bash_completion /usr/local/share/bash_completion/bkctld
# cp bkctld.conf /etc/default/bkctld
~~~
## Chroot dependencies

View file

@ -1,59 +0,0 @@
#!/bin/sh
#
# Description: check canary file
# Usage: check-canary [<jailname>|all]
#
# shellcheck source=./includes
LIBDIR="$(dirname $0)" && . "${LIBDIR}/includes"
return=0
nb_crit=0
nb_warn=0
nb_ok=0
nb_unkn=0
output=""
date=$(date +"%Y-%m-%d")
# Check each jail status
check_jail() {
jail_name=$1
jail_path=$(jail_path "${jail_name}")
canary_absolute_file="${jail_path}/var/backup/${CANARY_RELATIVE_FILE}"
if [ -f "${canary_absolute_file}" ]; then
if grep --quiet --fixed-string "${date}" "${canary_absolute_file}"; then
nb_ok=$((nb_ok + 1))
output="${output}OK - ${jail_name} - entries found for ${date} in ${CANARY_RELATIVE_FILE} file\n"
else
nb_crit=$((nb_crit + 1))
output="${output}CRITICAL - ${jail_name} - No entry for ${date} in ${CANARY_RELATIVE_FILE} file\n"
[ "${return}" -le 2 ] && return=2
fi
else
nb_crit=$((nb_crit + 1))
output="${output}CRITICAL - ${jail_name} - missing ${CANARY_RELATIVE_FILE} file\n"
[ "${return}" -le 2 ] && return=2
fi
}
for jail_name in $(jails_list); do
check_jail "${jail_name}"
done
[ "${return}" -ge 0 ] && header="OK"
[ "${return}" -ge 1 ] && header="WARNING"
[ "${return}" -ge 2 ] && header="CRITICAL"
[ "${return}" -ge 3 ] && header="UNKNOWN"
printf "%s - %s UNK / %s CRIT / %s WARN / %s OK\n\n" "${header}" "${nb_unkn}" "${nb_crit}" "${nb_warn}" "${nb_ok}"
printf "${output}" | grep -E "^UNKNOWN"
printf "${output}" | grep -E "^CRITICAL"
printf "${output}" | grep -E "^WARNING"
printf "${output}" | grep -E "^OK"
exit "${return}"

View file

@ -16,7 +16,7 @@ output=""
# Verify backup partition is mounted and writable
findmnt -O rw --mountpoint "${BACKUP_PARTITION}" > /dev/null
findmnt -O rw --target "${BACKUP_PARTITION}" > /dev/null
if [ "$?" -ne 0 ]; then
nb_crit=$((nb_crit + 1))
output="${output}CRITICAL - Backup disk \`/backup' is not mounted (or read-only) !\n"
@ -29,12 +29,11 @@ fi
# Check if the firewall file is sourced
minifirewall_config=/etc/default/minifirewall
minifirewall_version=$(/etc/init.d/minifirewall status | head -1 | cut -d ' ' -f 3)
if [ -n "${FIREWALL_RULES}" ] \
&& [ -r "${FIREWALL_RULES}" ] \
&& [ -f "${minifirewall_config}" ]; then
minifirewall_version=$(grep -E -o "^VERSION=(\S+)" /etc/init.d/minifirewall | head -1 | cut -d '=' -f 2 | tr -d "'" | tr -d '"')
if [ -n "${minifirewall_version}" ] && dpkg --compare-versions "${minifirewall_version}" ge "22.03"; then
# Minifirewall 22.03+ includes files automatically
nb_ok=$((nb_ok + 1))

View file

@ -51,15 +51,15 @@ if dry_run; then
else
mv "${jail_path}" "${new_jail_path}"
fi
if [ -d "${incs_path}" ]; then
if dry_run; then
if dry_run; then
if [ -d "${incs_path}" ]; then
echo "[dry-run] rename ${incs_path} to ${new_incs_path}"
else
fi
else
if [ -d "${incs_path}" ]; then
mv "${incs_path}" "${new_incs_path}"
fi
fi
if [ -d "${jail_config_dir}" ]; then
if dry_run; then
echo "[dry-run] rename ${jail_config_dir} to ${new_jail_config_dir}"

View file

@ -15,29 +15,27 @@ ionice -c3 "${DUC}" index -d "${IDX_FILE}" "${JAILDIR}"
touch "${INDEX_DIR}/.lastrun.duc"
EOF
[ ! -f "${INDEX_DIR}/.lastrun.duc" ] && notice "First run of DUC still in progress ..." && exit 0
[ ! -f "${INDEX_DIR}/.lastrun.duc" ] && notice "First run of DUC always in progress ..." && exit 0
[ ! -f ${IDX_FILE} ] && error "Index file doesn't exits !"
printf "Last update of index file : "
stat --format=%Y "${INDEX_DIR}/.lastrun.duc" | xargs -i -n1 date -R -d "@{}"
echo "<jail> <size> <incs> <lastconn>" | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $1, $2, $3, $4); }'
duc_output=$(mktemp)
stat_output=$(mktemp)
incs_output=$(mktemp)
jail_patterns_list=$(mktemp)
# shellcheck disable=SC2064
trap "rm ${duc_output} ${incs_output} ${stat_output} ${jail_patterns_list}" 0
trap "rm ${duc_output} ${incs_output} ${stat_output}" 0
"${DUC}" ls --database "${IDX_FILE}" "${JAILDIR}" > "${duc_output}"
"${DUC}" ls -d "${IDX_FILE}" "${JAILDIR}" > "${duc_output}"
jails_list | sed -e "s/^\(.*\)$/\\\\b\1\\\\b/" > "${jail_patterns_list}"
grep -f "${jail_patterns_list}" "${duc_output}" | awk '{ print $2 }' | while read jail_name; do
awk '{ print $2 }' "${duc_output}" | while read jail_name; do
jail_path=$(jail_path "${jail_name}")
stat --format=%Y "${jail_path}/var/log/lastlog" | xargs -i -n1 date -d "@{}" "+%d-%m-%Y" >> "${stat_output}"
incs_policy_file=$(current_jail_incs_policy_file "${jail_name}")
incs_policy_file=$(current_jail_incs_policy_file ${jail_name})
incs_policy="0"
if [ -r "${incs_policy_file}" ]; then
days=$(grep "^\+" "${incs_policy_file}" | grep --count "day")
@ -47,7 +45,4 @@ grep -f "${jail_patterns_list}" "${duc_output}" | awk '{ print $2 }' | while rea
echo "${incs_policy}" >> "${incs_output}"
done
(
echo "<jail> <size> <incs> <lastconn>"
paste "${duc_output}" "${incs_output}" "${stat_output}" | awk '{ printf("%s %s %s %s\n", $2, $1, $3, $4); }'
) | column -t
paste "${duc_output}" "${incs_output}" "${stat_output}" | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $2, $1, $3, $4); }'

View file

@ -1,7 +1,7 @@
#!/bin/sh
#
# Description: Display status of SSH server
# Usage: [--no-header] status [<jailname>|all]
# Usage: status [<jailname>|all]
#
# shellcheck source=./includes

View file

@ -6,7 +6,7 @@
[ -f /etc/default/bkctld ] && . /etc/default/bkctld
VERSION="22.11"
VERSION="22.04"
LIBDIR=${LIBDIR:-/usr/lib/bkctld}
CONFDIR="${CONFDIR:-/etc/evobackup}"
@ -20,7 +20,6 @@ LOCKDIR="${LOCKDIR:-/run/lock/bkctld}"
ARCHIVESDIR="${ARCHIVESDIR:-${BACKUP_PARTITION}/archives}"
INDEX_DIR="${INDEX_DIR:-${BACKUP_PARTITION}/index}"
IDX_FILE="${IDX_FILE:-${INDEX_DIR}/bkctld-jails.idx}"
CANARY_RELATIVE_FILE="${CANARY_RELATIVE_FILE:-/zzz_evobackup_canary}"
SSHD_PID="${SSHD_PID:-/run/sshd.pid}"
SSHD_CONFIG="${SSHD_CONFIG:-/etc/ssh/sshd_config}"
AUTHORIZED_KEYS="${AUTHORIZED_KEYS:-/root/.ssh/authorized_keys}"
@ -30,7 +29,6 @@ CRITICAL="${CRITICAL:-48}"
WARNING="${WARNING:-24}"
DUC=$(command -v duc-nox || command -v duc)
FORCE="${FORCE:-0}"
HEADER="${HEADER:-1}"
show_version() {
cat <<END
@ -64,13 +62,6 @@ EOF
printf "\n"
}
is_quiet() {
test ${QUIET} -eq 1
}
is_verbose() {
test ${VERBOSE} -eq 1
}
log_date() {
echo "[$(date +"%Y-%m-%d %H:%M:%S")]"
}
@ -136,7 +127,7 @@ is_btrfs() {
inode=$(stat --format=%i "${path}")
test "$inode" -eq 256
test $inode -eq 256
}
# Returns the list of jails found in the "jails" directory (default)

View file

@ -252,25 +252,3 @@ OUT
assert_failure
}
# TODO: write many more tests for bkctld-check-incs
@test "Check-canary fails if a canary file doesn't exist" {
run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}"
assert_equal "$status" "2"
assert_line "CRITICAL - ${JAILNAME} - missing /zzz_evobackup_canary file"
}
@test "Check-canary fails if a canary is missing today's entries" {
today="$(date +%Y-%m-%d)"
touch "${JAILPATH}/var/backup/zzz_evobackup_canary"
run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}"
assert_equal "$status" "2"
assert_line "CRITICAL - ${JAILNAME} - No entry for ${today} in /zzz_evobackup_canary file"
}
@test "Check-canary succeeds if a canary has today's entries" {
echo "$(date "+%FT%T%z") bats-test" >> "${JAILPATH}/var/backup/zzz_evobackup_canary"
run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}"
assert_success
}