diff --git a/roles/base/files/evomaintenance.sh b/roles/base/files/evomaintenance.sh index 1cd4ce7..3903f2e 100644 --- a/roles/base/files/evomaintenance.sh +++ b/roles/base/files/evomaintenance.sh @@ -4,16 +4,16 @@ # Dependencies (all OS): git postgresql-client # Dependencies (Debian): sudo -# Copyright 2007-2019 Evolix , Gregory Colpart , +# Copyright 2007-2022 Evolix , Gregory Colpart , # Jérémy Lecour and others. -VERSION="0.6.3" +VERSION="22.01" show_version() { cat <, +Copyright 2007-2022 Evolix , Gregory Colpart , Jérémy Lecour and others. @@ -178,9 +178,11 @@ is_repository_readonly() { if [ "$(get_system)" = "OpenBSD" ]; then partition=$(stat -f '%Sd' $1) mount | grep ${partition} | grep -q "read-only" - else + elif command -v findmnt >/dev/null; then mountpoint=$(stat -c '%m' $1) findmnt ${mountpoint} --noheadings --output OPTIONS -O ro + else + grep /usr /proc/mounts | grep -E '\bro\b' fi } remount_repository_readwrite() { @@ -301,6 +303,9 @@ From: ${FULLFROM} Content-Type: text/plain; charset=UTF-8 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit +X-Evomaintenance-Version: ${VERSION} +X-Evomaintenance-Host: ${HOSTNAME_TEXT} +X-Evomaintenance-User: ${USER} To: ${EVOMAINTMAIL} Subject: [evomaintenance] Intervention sur ${HOSTNAME_TEXT} (${USER}) diff --git a/roles/base/files/zzz_evobackup b/roles/base/files/zzz_evobackup index 094550b..d39c40a 100755 --- a/roles/base/files/zzz_evobackup +++ b/roles/base/files/zzz_evobackup @@ -20,6 +20,8 @@ set -u ##### Configuration ################################################### +VERSION="22.01" + # email adress for notifications MAIL=jdoe@example.com @@ -38,18 +40,23 @@ LOCAL_BACKUP_DIR="/home/backup" # You can set "linux" or "bsd" manually or let it choose automatically SYSTEM=$(uname | tr '[:upper:]' '[:lower:]') -# Store pid and logs in a file named after this program's name -PROGNAME=$(basename $0) +# Store pid in a file named after this program's name +PROGNAME=$(basename "$0") PIDFILE="/var/run/${PROGNAME}.pid" -LOGFILE="/var/log/${PROGNAME}.log" + +# Customize the log path if you have multiple scripts and with separate logs +LOGFILE="/var/log/evobackup.log" # Enable/Disable tasks LOCAL_TASKS=${LOCAL_TASKS:-1} SYNC_TASKS=${SYNC_TASKS:-1} +HOSTNAME=$(hostname) + ##### SETUP AND FUNCTIONS ############################################# -BEGINNING=$(/bin/date +"%d-%m-%Y ; %H:%M") +START_EPOCH=$(/bin/date +%s) +DATE_FORMAT="%Y-%m-%d %H:%M:%S" # shellcheck disable=SC2174 mkdir -p -m 700 ${LOCAL_BACKUP_DIR} @@ -84,6 +91,7 @@ test_server() { else # SSH connection failed new_error=$(printf "Failed to connect to \`%s' within %s seconds" "${item}" "${SSH_CONNECT_TIMEOUT}") + log "${new_error}" SERVERS_SSH_ERRORS=$(printf "%s\\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d') return 1 @@ -97,17 +105,16 @@ pick_server() { if [ "${increment}" -ge "${list_length}" ]; then # We've reached the end of the list new_error="No more server available" + log "${new_error}" SERVERS_SSH_ERRORS=$(printf "%s\\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d') # Log errors to stderr printf "%s\\n" "${SERVERS_SSH_ERRORS}" >&2 - # Log errors to logfile - printf "%s\\n" "${SERVERS_SSH_ERRORS}" >> $LOGFILE return 1 fi # Extract the day of month, without leading 0 (which would give an octal based number) - today=$(date +%e) + today=$(/bin/date +%e) # A salt is useful to randomize the starting point in the list # but stay identical each time it's called for a server (based on hostname). salt=$(hostname | cksum | cut -d' ' -f1) @@ -119,6 +126,15 @@ pick_server() { echo "${SERVERS}" | cut -d' ' -f${field} } +log() { + msg="${1:-$(cat /dev/stdin)}" + pid=$$ + printf "[%s] %s[%s]: %s\\n" \ + "$(/bin/date +"${DATE_FORMAT}")" "${PROGNAME}" "${pid}" "${msg}" \ + >> "${LOGFILE}" +} + +log "START GLOBAL - VERSION=${VERSION} LOCAL_TASKS=${LOCAL_TASKS} SYNC_TASKS=${SYNC_TASKS}" ## Verify other evobackup process and kill if needed if [ -e "${PIDFILE}" ]; then @@ -133,16 +149,18 @@ if [ -e "${PIDFILE}" ]; then kill -9 "${pid}" printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\\n" >&2 else - rm -f ${PIDFILE} + rm -f "${PIDFILE}" fi fi -echo "$$" > ${PIDFILE} +echo "$$" > "${PIDFILE}" # shellcheck disable=SC2064 trap "rm -f ${PIDFILE}" EXIT ##### LOCAL BACKUP #################################################### if [ "${LOCAL_TASKS}" = "1" ]; then + log "START LOCAL_TASKS" + # You can comment or uncomment sections below to customize the backup ## OpenLDAP : example with slapcat @@ -152,16 +170,22 @@ if [ "${LOCAL_TASKS}" = "1" ]; then ## MySQL + ## Purge previous dumps + # rm -f ${LOCAL_BACKUP_DIR}/mysql.*.gz + # rm -rf ${LOCAL_BACKUP_DIR}/mysql + # rm -rf ${LOCAL_BACKUP_DIR}/mysqlhotcopy + # rm -rf /home/mysqldump + ## example with global and compressed mysqldump # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ # --opt --all-databases --force --events --hex-blob | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz - ## example with two dumps for each table (.sql/.txt) for all databases - # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ - # | egrep -v "^(Database|information_schema|performance_schema|sys)" ); \ - # do mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump ; \ - # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \ - # --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i; done + ## example with compressed SQL dump (with data) for each databases + # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/ + # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \ + # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do + # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i | gzip --best > ${LOCAL_BACKUP_DIR}/mysql/${i}.sql.gz + # done ## Dump all grants (requires 'percona-toolkit' package) # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/ @@ -174,22 +198,22 @@ if [ "${LOCAL_TASKS}" = "1" ]; then # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases $i > ${LOCAL_BACKUP_DIR}/mysql/${i}.schema.sql # done - ## example with compressed SQL dump (with data) for each databases - # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/ - # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \ - # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do - # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i | gzip --best > ${LOCAL_BACKUP_DIR}/mysql/${i}.sql.gz - # done - ## example with *one* uncompressed SQL dump for *one* database (MYBASE) # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/MYBASE # chown -RL mysql ${LOCAL_BACKUP_DIR}/mysql/ # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -Q \ # --opt --events --hex-blob --skip-comments -T ${LOCAL_BACKUP_DIR}/mysql/MYBASE MYBASE + ## example with two dumps for each table (.sql/.txt) for all databases + # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \ + # | egrep -v "^(Database|information_schema|performance_schema|sys)" ); \ + # do mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump ; \ + # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \ + # --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i; done + ## example with mysqlhotcopy # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysqlhotcopy/ - # mysqlhotcopy BASE ${LOCAL_BACKUP_DIR}/mysql/mysqlhotcopy/ + # mysqlhotcopy MYBASE ${LOCAL_BACKUP_DIR}/mysqlhotcopy/ ## example for multiples MySQL instances # mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3) @@ -197,12 +221,16 @@ if [ "${LOCAL_TASKS}" = "1" ]; then # instance=$(echo "$instance"|awk '{ print $3 }') # if [ "$instance" != "3306" ] # then - # mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd > ${LOCAL_BACKUP_DIR}/mysql.$instance.bak + # mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.$instance.bak.gz # fi # done ## PostgreSQL + ## Purge previous dumps + # rm ${LOCAL_BACKUP_DIR}/pg.*.gz + # rm ${LOCAL_BACKUP_DIR}/pg-backup.tar + # rm ${LOCAL_BACKUP_DIR}/postgresql/* ## example with pg_dumpall (warning: you need space in ~postgres) # su - postgres -c "pg_dumpall > ~/pg.dump.bak" # mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/ @@ -217,12 +245,20 @@ if [ "${LOCAL_TASKS}" = "1" ]; then ## example with only TABLE1 and TABLE2 from MYBASE # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE + ## example with compressed PostgreSQL dump for each databases + # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/postgresql + # chown postgres:postgres ${LOCAL_BACKUP_DIR}/postgresql + # dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*) + # + # for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done + ## MongoDB ## don't forget to create use with read-only access ## > use admin ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } ) - # test -d ${LOCAL_BACKUP_DIR}/mongodump/ && rm -rf ${LOCAL_BACKUP_DIR}/mongodump/ + ## Purge previous dumps + # rm -rf ${LOCAL_BACKUP_DIR}/mongodump/ # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/ # mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/ # if [ $? -ne 0 ]; then @@ -231,9 +267,13 @@ if [ "${LOCAL_TASKS}" = "1" ]; then ## Redis + ## Purge previous dumps + # rm -rf ${LOCAL_BACKUP_DIR}/redis/ + # rm -rf ${LOCAL_BACKUP_DIR}/redis-* ## example with copy .rdb file ## for the default instance : - # cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/ + # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/redis/ + # cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/redis/ ## for multiple instances : # for instance in $(ls -d /var/lib/redis-*); do # name=$(basename $instance) @@ -261,7 +301,7 @@ if [ "${LOCAL_TASKS}" = "1" ]; then # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}' # done - # date=$(date +%F) + # date=$(/bin/date +%F) # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log ## RabbitMQ @@ -273,104 +313,126 @@ if [ "${LOCAL_TASKS}" = "1" ]; then #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null - ## Dump system and kernel versions - uname -a > ${LOCAL_BACKUP_DIR}/uname - ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr} traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1 done - ## Dump process with ps - ps auwwx >${LOCAL_BACKUP_DIR}/ps.out + server_state_dir="${LOCAL_BACKUP_DIR}/server-state" + + backup_server_state_bin=$(command -v backup-server-state) if [ "${SYSTEM}" = "linux" ]; then - ## Dump network connections with ss - ss -taupen > ${LOCAL_BACKUP_DIR}/netstat.out + if [ -n "${backup_server_state_bin}" ]; then + ${backup_server_state_bin} --etc --dpkg-full --force --backup-dir "${server_state_dir}" + else + mkdir -p "${server_state_dir}" - ## List Debian packages - dpkg -l > ${LOCAL_BACKUP_DIR}/packages - dpkg --get-selections > ${LOCAL_BACKUP_DIR}/packages.getselections - apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available + ## Dump system and kernel versions + uname -a > ${server_state_dir}/uname.txt - ## Dump MBR / table partitions - disks=$(lsblk -l | grep disk | grep -v -E '(drbd|fd[0-9]+)' | awk '{print $1}') - for disk in ${disks}; do - dd if="/dev/${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)" - fdisk -l "/dev/${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${disk}" 2>&1 - done - cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions + ## Dump process with ps + ps auwwx > ${server_state_dir}/ps.txt - ## Dump iptables - if [ -x /sbin/iptables ]; then - { /sbin/iptables -L -n -v; /sbin/iptables -t filter -L -n -v; } > ${LOCAL_BACKUP_DIR}/iptables.txt - fi + ## Dump network connections with ss + ss -taupen > ${server_state_dir}/netstat.txt - ## Dump findmnt(8) output - FINDMNT_BIN=$(command -v findmnt) - if [ -x "${FINDMNT_BIN}" ]; then - ${FINDMNT_BIN} > ${LOCAL_BACKUP_DIR}/findmnt.txt + ## List Debian packages + dpkg -l > ${server_state_dir}/packages + dpkg --get-selections > ${server_state_dir}/packages.getselections + apt-cache dumpavail > ${server_state_dir}/packages.available + + ## Dump iptables + if [ -x /sbin/iptables ]; then + { /sbin/iptables -L -n -v; /sbin/iptables -t filter -L -n -v; } > ${server_state_dir}/iptables.txt + fi + + ## Dump findmnt(8) output + FINDMNT_BIN=$(command -v findmnt) + if [ -x "${FINDMNT_BIN}" ]; then + ${FINDMNT_BIN} > ${server_state_dir}/findmnt.txt + fi + + ## Dump MBR / table partitions + disks=$(lsblk -l | grep disk | grep -v -E '(drbd|fd[0-9]+)' | awk '{print $1}') + for disk in ${disks}; do + dd if="/dev/${disk}" of="${server_state_dir}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)" + fdisk -l "/dev/${disk}" > "${server_state_dir}/partitions-${disk}" 2>&1 + done + cat ${server_state_dir}/partitions-* > ${server_state_dir}/partitions fi else - ## Dump network connections with fstat - fstat | head -1 > ${LOCAL_BACKUP_DIR}/netstat.out - fstat | grep internet >> ${LOCAL_BACKUP_DIR}/netstat.out + if [ -n "${backup_server_state_bin}" ]; then + ${backup_server_state_bin} --force --backup-dir "${server_state_dir}" + else + mkdir -p "${server_state_dir}" - ## List OpenBSD packages - pkg_info -m > ${LOCAL_BACKUP_DIR}/packages + ## Dump system and kernel versions + uname -a > ${server_state_dir}/uname - ## Dump MBR / table partitions - disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions + ## Dump process with ps + ps auwwx > ${server_state_dir}/ps.out - ## Dump pf infos - pfctl -sa > ${LOCAL_BACKUP_DIR}/pfctl-sa.txt + ## Dump network connections with fstat + fstat | head -1 > ${server_state_dir}/netstat.out + fstat | grep internet >> ${server_state_dir}/netstat.out + ## List OpenBSD packages + pkg_info -m > ${server_state_dir}/packages + + ## Dump MBR / table partitions + disklabel sd0 > ${server_state_dir}/partitions + + ## Dump pf infos + pfctl -sa > ${server_state_dir}/pfctl-sa.txt + fi fi ## Dump rights - #getfacl -R /var > ${LOCAL_BACKUP_DIR}/rights-var.txt - #getfacl -R /etc > ${LOCAL_BACKUP_DIR}/rights-etc.txt - #getfacl -R /usr > ${LOCAL_BACKUP_DIR}/rights-usr.txt - #getfacl -R /home > ${LOCAL_BACKUP_DIR}/rights-home.txt + #getfacl -R /var > ${server_state_dir}/rights-var.txt + #getfacl -R /etc > ${server_state_dir}/rights-etc.txt + #getfacl -R /usr > ${server_state_dir}/rights-usr.txt + #getfacl -R /home > ${server_state_dir}/rights-home.txt + log "STOP LOCAL_TASKS" fi ##### REMOTE BACKUP ################################################### -n=0 -server="" -if [ "${SERVERS_FALLBACK}" = "1" ]; then - # We try to find a suitable server - while :; do - server=$(pick_server "${n}") - test $? = 0 || exit 2 - - if test_server "${server}"; then - break - else - server="" - n=$(( n + 1 )) - fi - done -else - # we force the server - server=$(pick_server "${n}") -fi - -SSH_SERVER=$(echo "${server}" | cut -d':' -f1) -SSH_PORT=$(echo "${server}" | cut -d':' -f2) - -HOSTNAME=$(hostname) - -if [ "${SYSTEM}" = "linux" ]; then - rep="/bin /boot /lib /opt /sbin /usr" -else - rep="/bsd /bin /sbin /usr" -fi - if [ "${SYNC_TASKS}" = "1" ]; then + n=0 + server="" + if [ "${SERVERS_FALLBACK}" = "1" ]; then + # We try to find a suitable server + while :; do + server=$(pick_server "${n}") + test $? = 0 || exit 2 + + if test_server "${server}"; then + break + else + server="" + n=$(( n + 1 )) + fi + done + else + # we force the server + server=$(pick_server "${n}") + fi + + SSH_SERVER=$(echo "${server}" | cut -d':' -f1) + SSH_PORT=$(echo "${server}" | cut -d':' -f2) + + if [ "${SYSTEM}" = "linux" ]; then + rep="/bin /boot /lib /opt /sbin /usr" + else + rep="/bsd /bin /sbin /usr" + fi + + log "START SYNC_TASKS - server=${server}" + # /!\ DO NOT USE COMMENTS in the rsync command /!\ # It breaks the command and destroys data, simply remove (or add) lines. @@ -403,13 +465,13 @@ if [ "${SYNC_TASKS}" = "1" ]; then --exclude "/var/lib/postgresql" \ --exclude "/var/lib/sympa" \ --exclude "/var/lock" \ - --exclude "/var/log" \ - --exclude "/var/log/evobackup*" \ --exclude "/var/run" \ --exclude "/var/spool/postfix" \ --exclude "/var/spool/smtpd" \ --exclude "/var/spool/squid" \ --exclude "/var/state" \ + --exclude "/var/tmp" \ + --exclude "lxc/*/rootfs/tmp" \ --exclude "lxc/*/rootfs/usr/doc" \ --exclude "lxc/*/rootfs/usr/obj" \ --exclude "lxc/*/rootfs/usr/share/doc" \ @@ -422,6 +484,7 @@ if [ "${SYNC_TASKS}" = "1" ]; then --exclude "lxc/*/rootfs/var/log" \ --exclude "lxc/*/rootfs/var/run" \ --exclude "lxc/*/rootfs/var/state" \ + --exclude "lxc/*/rootfs/var/tmp" \ --exclude "/home/mysqltmp" \ ${rep} \ /etc \ @@ -431,20 +494,19 @@ if [ "${SYNC_TASKS}" = "1" ]; then -e "${RSH_COMMAND}" \ "root@${SSH_SERVER}:/var/backup/" \ | tail -30 >> $LOGFILE + + log "STOP SYNC_TASKS - server=${server}" fi ##### REPORTING ####################################################### -END=$(/bin/date +"%d-%m-%Y ; %H:%M") +STOP_EPOCH=$(/bin/date +%s) -printf "EvoBackup - %s - START %s ON %s (LOCAL_TASKS=%s SYNC_TASKS=%s)\\n" \ - "${HOSTNAME}" "${BEGINNING}" "${SSH_SERVER}" "${LOCAL_TASKS}" "${SYNC_TASKS}" \ - >> $LOGFILE +start_time=$(/bin/date --date="@${START_EPOCH}" +"${DATE_FORMAT}") +stop_time=$(/bin/date --date="@${STOP_EPOCH}" +"${DATE_FORMAT}") +duration=$(( STOP_EPOCH - START_EPOCH )) -printf "EvoBackup - %s - STOP %s ON %s (LOCAL_TASKS=%s SYNC_TASKS=%s)\\n" \ - "${HOSTNAME}" "${END}" "${SSH_SERVER}" "${LOCAL_TASKS}" "${SYNC_TASKS}" \ - >> $LOGFILE +log "STOP GLOBAL - start='${start_time}' stop='${stop_time}' duration=${duration}s" -tail -10 $LOGFILE | \ - mail -s "[info] EvoBackup - Client ${HOSTNAME}" \ - ${MAIL} +tail -20 "${LOGFILE}" \ + | mail -s "[info] EvoBackup - Client ${HOSTNAME}" ${MAIL}