evobackup/zzz_evobackup

#!/bin/sh
#
# Script Evobackup client
# See https://gitea.evolix.org/evolix/evobackup
#
# Author: Gregory Colpart <reg@evolix.fr>
# Contributors:
# Romain Dessort <rdessort@evolix.fr>
# Benoît Série <bserie@evolix.fr>
# Tristan Pilat <tpilat@evolix.fr>
# Victor Laborie <vlaborie@evolix.fr>
# Jérémy Lecour <jlecour@evolix.fr>
#
# Licence: AGPLv3
#
# /!\ DON'T FORGET TO SET "MAIL" and "SERVERS" VARIABLES

# Fail on unassigned variables
set -u

##### Configuration ###################################################

VERSION="22.03"

# email adress for notifications
MAIL=jdoe@example.com

# list of hosts (hostname or IP) and SSH port for Rsync
SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX"

# Should we fallback on servers when the first is unreachable ?
SERVERS_FALLBACK=${SERVERS_FALLBACK:-1}

# timeout (in seconds) for SSH connections
SSH_CONNECT_TIMEOUT=${SSH_CONNECT_TIMEOUT:-90}

# We use /home/backup : feel free to use your own dir
LOCAL_BACKUP_DIR="/home/backup"

# You can set "linux" or "bsd" manually or let it choose automatically
SYSTEM=$(uname | tr '[:upper:]' '[:lower:]')

# Store pid in a file named after this program's name
PROGNAME=$(basename "$0")
PIDFILE="/var/run/${PROGNAME}.pid"

# Customize the log path if you have multiple scripts and with separate logs
LOGFILE="/var/log/evobackup.log"

# Enable/Disable tasks
LOCAL_TASKS=${LOCAL_TASKS:-1}
SYNC_TASKS=${SYNC_TASKS:-1}

HOSTNAME=$(hostname)

##### SETUP AND FUNCTIONS #############################################

START_EPOCH=$(/bin/date +%s)
DATE_FORMAT="%Y-%m-%d %H:%M:%S"

# shellcheck disable=SC2174
mkdir -p -m 700 ${LOCAL_BACKUP_DIR}

PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin

## lang = C for english outputs
export LANGUAGE=C
export LANG=C

## Force umask
umask 077

## Initialize variable to store SSH connection errors
SERVERS_SSH_ERRORS=""

# Call test_server with "HOST:PORT" string
# It will return with 0 if the server is reachable.
# It will return with 1 and a message on stderr if not.
test_server() {
    item=$1
    # split HOST and PORT from the input string
    host=$(echo "${item}" | cut -d':' -f1)
    port=$(echo "${item}" | cut -d':' -f2)

    # Test if the server is accepting connections
    ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" "${host}" -p "${port}" -t "exit"
    # shellcheck disable=SC2181
    if [ $? = 0 ]; then
        # SSH connection is OK
        return 0
    else
        # SSH connection failed
        new_error=$(printf "Failed to connect to \`%s' within %s seconds" "${item}" "${SSH_CONNECT_TIMEOUT}")
        log "${new_error}"
        SERVERS_SSH_ERRORS=$(printf "%s\\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d')

        return 1
    fi
}
# Call pick_server with an optional positive integer to get the nth server in the list.
pick_server() {
    increment=${1:-0}
    list_length=$(echo "${SERVERS}" | wc -w)

    if [ "${increment}" -ge "${list_length}" ]; then
        # We've reached the end of the list
        new_error="No more server available"
        log "${new_error}"
        SERVERS_SSH_ERRORS=$(printf "%s\\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d')

        # Log errors to stderr
        printf "%s\\n" "${SERVERS_SSH_ERRORS}" >&2
        return 1
    fi

    # Extract the day of month, without leading 0 (which would give an octal based number)
    today=$(/bin/date +%e)
    # A salt is useful to randomize the starting point in the list
    # but stay identical each time it's called for a server (based on hostname).
    salt=$(hostname | cksum | cut -d' ' -f1)
    # Pick an integer between 0 and the length of the SERVERS list
    # It changes each day
    item=$(( (today + salt + increment) % list_length ))
    # cut starts counting fields at 1, not 0.
    field=$(( item + 1 ))

    echo "${SERVERS}" | cut -d' ' -f${field}
}
log() {
    msg="${1:-$(cat /dev/stdin)}"
    pid=$$
    printf "[%s] %s[%s]: %s\\n" \
        "$(/bin/date +"${DATE_FORMAT}")" "${PROGNAME}" "${pid}" "${msg}" \
        >> "${LOGFILE}"
}

log "START GLOBAL - VERSION=${VERSION} LOCAL_TASKS=${LOCAL_TASKS} SYNC_TASKS=${SYNC_TASKS}"

## Verify other evobackup process and kill if needed
if [ -e "${PIDFILE}" ]; then
    pid=$(cat "${PIDFILE}")
    # Does process still exist ?
    if kill -0 "${pid}" 2> /dev/null; then
        # Killing the childs of evobackup.
        for ppid in $(pgrep -P "${pid}"); do
            kill -9 "${ppid}";
        done
        # Then kill the main PID.
        kill -9 "${pid}"
        printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\\n" >&2
    else
        rm -f "${PIDFILE}"
    fi
fi
echo "$$" > "${PIDFILE}"
# shellcheck disable=SC2064
trap "rm -f ${PIDFILE}" EXIT

##### LOCAL BACKUP ####################################################

if [ "${LOCAL_TASKS}" = "1" ]; then
    log "START LOCAL_TASKS"

    # You can comment or uncomment sections below to customize the backup

    ## OpenLDAP : example with slapcat
    # slapcat -n 0 -l ${LOCAL_BACKUP_DIR}/config.ldap.bak
    # slapcat -n 1 -l ${LOCAL_BACKUP_DIR}/data.ldap.bak
    # slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak

    ## MySQL

    ## Purge previous dumps
    # rm -f  ${LOCAL_BACKUP_DIR}/mysql.*.gz
    # rm -rf ${LOCAL_BACKUP_DIR}/mysql
    # rm -rf ${LOCAL_BACKUP_DIR}/mysqlhotcopy
    # rm -rf /home/mysqldump

    ## example with global and compressed mysqldump
    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
    #  --opt --all-databases --force --events --hex-blob | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz

    ## example with compressed SQL dump (with data) for each databases
    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
    # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
    # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i | gzip --best > ${LOCAL_BACKUP_DIR}/mysql/${i}.sql.gz
    # done

    ## Dump all grants (requires 'percona-toolkit' package)
    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
    # pt-show-grants --flush --no-header > ${LOCAL_BACKUP_DIR}/mysql/all_grants.sql

    # Dump all variables
    # mysql -A -e"SHOW GLOBAL VARIABLES;" > ${LOCAL_BACKUP_DIR}/MySQLCurrentSettings.txt

    ## example with SQL dump (schema only, no data) for each databases
    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
    # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
    # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases $i > ${LOCAL_BACKUP_DIR}/mysql/${i}.schema.sql
    # done

    ## example with *one* uncompressed SQL dump for *one* database (MYBASE)
    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/MYBASE
    # chown -RL mysql ${LOCAL_BACKUP_DIR}/mysql/
    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -Q \
    # --opt --events --hex-blob --skip-comments -T ${LOCAL_BACKUP_DIR}/mysql/MYBASE MYBASE

    ## example with two dumps for each table (.sql/.txt) for all databases
    # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
    # | egrep -v "^(Database|information_schema|performance_schema|sys)" ); \
    # do mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump ; \
    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \
    #  --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i; done

    ## example with mysqlhotcopy
    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysqlhotcopy/
    # mysqlhotcopy MYBASE ${LOCAL_BACKUP_DIR}/mysqlhotcopy/

    ## example for multiples MySQL instances
    # mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3)
    # grep -E "^port\s*=\s*\d*" /etc/mysql/my.cnf |while read instance; do
    #     instance=$(echo "$instance"|awk '{ print $3 }')
    #     if [ "$instance" != "3306" ]
    #     then
    #         mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.$instance.bak.gz
    #     fi
    # done

    ## PostgreSQL

    ## Purge previous dumps
    # rm -rf ${LOCAL_BACKUP_DIR}/pg.*.gz
    # rm -rf ${LOCAL_BACKUP_DIR}/pg-backup.tar
    # rm -rf ${LOCAL_BACKUP_DIR}/postgresql/*
    ## example with pg_dumpall (warning: you need space in ~postgres)
    # su - postgres -c "pg_dumpall > ~/pg.dump.bak"
    # mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/
    ## another method with gzip directly piped
    # cd /var/lib/postgresql
    # sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
    # cd - > /dev/null

    ## example with all tables from MYBASE excepts TABLE1 and TABLE2
    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE

    ## example with only TABLE1 and TABLE2 from MYBASE
    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE

    ## example with compressed PostgreSQL dump for each databases
    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/postgresql
    # chown postgres:postgres ${LOCAL_BACKUP_DIR}/postgresql
    # dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*)
    # 
    # for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done

    ## MongoDB

    ## don't forget to create use with read-only access
    ## > use admin
    ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } )
    ## Purge previous dumps
    # rm -rf ${LOCAL_BACKUP_DIR}/mongodump/
    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/
    # mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/
    # if [ $? -ne 0 ]; then
    #   echo "Error with mongodump!"
    # fi

    ## Redis

    ## Purge previous dumps
    # rm -rf ${LOCAL_BACKUP_DIR}/redis/
    # rm -rf ${LOCAL_BACKUP_DIR}/redis-*
    ## example with copy .rdb file
    ## for the default instance :
    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/redis/
    # cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/redis/
    ## for multiple instances :
    # for instance in $(ls -d /var/lib/redis-*); do
    #     name=$(basename $instance)
    #     mkdir -p ${LOCAL_BACKUP_DIR}/${name}
    #     cp -a ${instance}/dump.rdb ${LOCAL_BACKUP_DIR}/${name}
    # done

    ## ElasticSearch

    ## Take a snapshot as a backup.
    ## Warning: You need to have a path.repo configured.
    ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes
    # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log
    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log
    ## Clustered version here
    ## It basically the same thing except that you need to check that NFS is mounted
    # if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2'
    # then
    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log
    #     curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log
    # else
    #     echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
    # fi
    ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
    # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
    # done
    # date=$(/bin/date +%F)
    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log

    ## RabbitMQ

    ## export config
    #rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> $LOGFILE

    ## MegaCli config

    #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null

    ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls)
    for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do
        mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr}
        traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1
    done

    server_state_dir="${LOCAL_BACKUP_DIR}/server-state"

    backup_server_state_bin=$(command -v backup-server-state)

    if [ "${SYSTEM}" = "linux" ]; then
        if [ -n "${backup_server_state_bin}" ]; then
            ${backup_server_state_bin} --etc --dpkg-full --force --backup-dir "${server_state_dir}"
        else
            mkdir -p "${server_state_dir}"

            ## Dump system and kernel versions
            uname -a > ${server_state_dir}/uname.txt

            ## Dump process with ps
            ps auwwx > ${server_state_dir}/ps.txt

            ## Dump network connections with ss
            ss -taupen > ${server_state_dir}/netstat.txt

            ## List Debian packages
            dpkg -l > ${server_state_dir}/packages
            dpkg --get-selections > ${server_state_dir}/packages.getselections
            apt-cache dumpavail > ${server_state_dir}/packages.available

            ## Dump iptables
            if [ -x /sbin/iptables ]; then
                { /sbin/iptables -L -n -v; /sbin/iptables -t filter -L -n -v; } > ${server_state_dir}/iptables.txt
            fi

            ## Dump findmnt(8) output
            FINDMNT_BIN=$(command -v findmnt)
            if [ -x "${FINDMNT_BIN}" ]; then
                ${FINDMNT_BIN} > ${server_state_dir}/findmnt.txt
            fi

            ## Dump MBR / table partitions
            disks=$(lsblk -l | grep disk | grep -v -E '(drbd|fd[0-9]+)' | awk '{print $1}')
            for disk in ${disks}; do
                dd if="/dev/${disk}" of="${server_state_dir}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)"
                fdisk -l "/dev/${disk}" > "${server_state_dir}/partitions-${disk}" 2>&1
            done
            cat ${server_state_dir}/partitions-* > ${server_state_dir}/partitions
        fi
    else
        if [ -n "${backup_server_state_bin}" ]; then
            ${backup_server_state_bin} --etc --force --backup-dir "${server_state_dir}"
        else
            mkdir -p "${server_state_dir}"

            ## Dump system and kernel versions
            uname -a > ${server_state_dir}/uname

            ## Dump process with ps
            ps auwwx > ${server_state_dir}/ps.out

            ## Dump network connections with fstat
            fstat | head -1 > ${server_state_dir}/netstat.out
            fstat | grep internet >> ${server_state_dir}/netstat.out

            ## List OpenBSD packages
            pkg_info -m > ${server_state_dir}/packages

            ## Dump MBR / table partitions
            disklabel sd0 > ${server_state_dir}/partitions

            ## Dump pf infos
            pfctl -sa > ${server_state_dir}/pfctl-sa.txt
        fi
    fi

    ## Dump rights
    #getfacl -R /var  > ${server_state_dir}/rights-var.txt
    #getfacl -R /etc  > ${server_state_dir}/rights-etc.txt
    #getfacl -R /usr  > ${server_state_dir}/rights-usr.txt
    #getfacl -R /home > ${server_state_dir}/rights-home.txt

    log "STOP LOCAL_TASKS"
fi

##### REMOTE BACKUP ###################################################


if [ "${SYNC_TASKS}" = "1" ]; then
    n=0
    server=""
    if [ "${SERVERS_FALLBACK}" = "1" ]; then
        # We try to find a suitable server
        while :; do
            server=$(pick_server "${n}")
            test $? = 0 || exit 2

            if test_server "${server}"; then
                break
            else
                server=""
                n=$(( n + 1 ))
            fi
        done
    else
        # we force the server
        server=$(pick_server "${n}")
    fi

    SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
    SSH_PORT=$(echo "${server}" | cut -d':' -f2)

    if [ "${SYSTEM}" = "linux" ]; then
        rep="/bin /boot /lib /opt /sbin /usr"
    else
        rep="/bsd /bin /sbin /usr"
    fi

    log "START SYNC_TASKS - server=${server}"

    # /!\ DO NOT USE COMMENTS in the rsync command /!\
    # It breaks the command and destroys data, simply remove (or add) lines.

    # Remote shell command
    RSH_COMMAND="ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'"

    # ignore check because we want it to split the different arguments to $rep
    # shellcheck disable=SC2086
    rsync -avzh --relative --stats --delete --delete-excluded --force --ignore-errors --partial \
        --exclude "dev"                                    \
        --exclude "lost+found"                             \
        --exclude ".nfs.*"                                 \
        --exclude "/usr/doc"                               \
        --exclude "/usr/obj"                               \
        --exclude "/usr/share/doc"                         \
        --exclude "/usr/src"                               \
        --exclude "/var/apt"                               \
        --exclude "/var/cache"                             \
        --exclude "/var/lib/amavis/amavisd.sock"           \
        --exclude "/var/lib/amavis/tmp"                    \
        --exclude "/var/lib/clamav/*.tmp"                  \
        --exclude "/var/lib/elasticsearch"                 \
        --exclude "/var/lib/metche"                        \
        --exclude "/var/lib/munin/*tmp*"                   \
        --exclude "/var/db/munin/*.tmp"                    \
        --exclude "/var/lib/mysql"                         \
        --exclude "/var/lib/php5"                          \
        --exclude "/var/lib/php/sessions"                  \
        --exclude "/var/lib/postgres"                      \
        --exclude "/var/lib/postgresql"                    \
        --exclude "/var/lib/sympa"                         \
        --exclude "/var/lock"                              \
        --exclude "/var/run"                               \
        --exclude "/var/spool/postfix"                     \
        --exclude "/var/spool/smtpd"                       \
        --exclude "/var/spool/squid"                       \
        --exclude "/var/state"                             \
        --exclude "/var/tmp"                               \
        --exclude "lxc/*/rootfs/tmp"                       \
        --exclude "lxc/*/rootfs/usr/doc"                   \
        --exclude "lxc/*/rootfs/usr/obj"                   \
        --exclude "lxc/*/rootfs/usr/share/doc"             \
        --exclude "lxc/*/rootfs/usr/src"                   \
        --exclude "lxc/*/rootfs/var/apt"                   \
        --exclude "lxc/*/rootfs/var/cache"                 \
        --exclude "lxc/*/rootfs/var/lib/php5"              \
        --exclude "lxc/*/rootfs/var/lib/php/sessions"      \
        --exclude "lxc/*/rootfs/var/lock"                  \
        --exclude "lxc/*/rootfs/var/log"                   \
        --exclude "lxc/*/rootfs/var/run"                   \
        --exclude "lxc/*/rootfs/var/state"                 \
        --exclude "lxc/*/rootfs/var/tmp"                   \
        --exclude "/home/mysqltmp"                         \
            ${rep}                                         \
            /etc                                           \
            /root                                          \
            /var                                           \
            /home                                          \
        -e "${RSH_COMMAND}"                                \
        "root@${SSH_SERVER}:/var/backup/"                  \
        | tail -30 >> $LOGFILE

    log "STOP SYNC_TASKS - server=${server}"
fi

##### REPORTING #######################################################

STOP_EPOCH=$(/bin/date +%s)

start_time=$(/bin/date --date="@${START_EPOCH}" +"${DATE_FORMAT}")
stop_time=$(/bin/date --date="@${STOP_EPOCH}" +"${DATE_FORMAT}")
duration=$(( STOP_EPOCH - START_EPOCH ))

log "STOP GLOBAL - start='${start_time}' stop='${stop_time}' duration=${duration}s"

tail -20 "${LOGFILE}" \
    | mail -s "[info] EvoBackup - Client ${HOSTNAME}" ${MAIL}