#!/bin/bash
#
# Script Evobackup client
# See https://gitea.evolix.org/evolix/evobackup
#
# Authors: Evolix <info@evolix.fr>,
#          Gregory Colpart <reg@evolix.fr>,
#          Romain Dessort <rdessort@evolix.fr>,
#          Benoit Série <bserie@evolix.fr>,
#          Tristan Pilat <tpilat@evolix.fr>,
#          Victor Laborie <vlaborie@evolix.fr>,
#          Jérémy Lecour <jlecour@evolix.fr>
#          and others.
#
# Licence: AGPLv3

VERSION="22.12"

##### CONFIGURATION ###################################################
#
# 1. Set the following MAIL and SERVERS variables.
# 2. Customize the RSYNC_INCLUDES and RSYNC_EXCLUDES variables.
# 3. Enable or disable local tasks inside the local_tasks() function.
#
# Some local tasks are configurable.
# If you enable them, have a look at their implementation.
#
# Some additional configuration variable can be customized
# at the end of the script, before invoking the main() function.
#
#######################################################################

# email adress for notifications
MAIL=jdoe@example.com

# list of hosts (hostname or IP) and SSH port for Rsync
SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX"

# We use /home/backup : feel free to use your own dir
LOCAL_BACKUP_DIR="/home/backup"

# Enable/disable local tasks (default: enabled)
: "${LOCAL_TASKS:=1}"
# Enable/disable sync tasks (default: enabled)
: "${SYNC_TASKS:=1}"

# Source paths can be customized
# Empty lines, and lines containing # or ; are ignored
RSYNC_INCLUDES="
/etc
/root
/var
/home
"

# Excluded paths can be customized
# Empty lines, and lines beginning with # or ; are ignored
RSYNC_EXCLUDES="
/dev
/proc
/run
/sys
/tmp
/usr/doc
/usr/obj
/usr/share/doc
/usr/src
/var/apt
/var/cache
/var/db/munin/*.tmp
/var/lib/amavis/amavisd.sock
/var/lib/amavis/tmp
/var/lib/clamav/*.tmp
/var/lib/elasticsearch
/var/lib/metche
/var/lib/mongodb
/var/lib/munin/*tmp*
/var/lib/mysql
/var/lib/php/sessions
/var/lib/php5
/var/lib/postgres
/var/lib/postgresql
/var/lib/sympa
/var/lock
/var/run
/var/spool/postfix
/var/spool/smtpd
/var/spool/squid
/var/state
/var/tmp
lost+found
.nfs.*
lxc/*/rootfs/tmp
lxc/*/rootfs/usr/doc
lxc/*/rootfs/usr/obj
lxc/*/rootfs/usr/share/doc
lxc/*/rootfs/usr/src
lxc/*/rootfs/var/apt
lxc/*/rootfs/var/cache
lxc/*/rootfs/var/lib/php5
lxc/*/rootfs/var/lib/php/sessions
lxc/*/rootfs/var/lock
lxc/*/rootfs/var/run
lxc/*/rootfs/var/state
lxc/*/rootfs/var/tmp
/home/mysqltmp
"

##### FUNCTIONS #######################################################

# Execute all local tasks: database dumps, system state dump…
local_tasks() {
    log "START LOCAL_TASKS"

    # Remove previous error files
    find "${LOCAL_BACKUP_DIR}/" -type f -name '*.err' -delete

    ###################################################################
    # You can enable/disable local tasks
    # by (un)commenting calls to "dump_XXX" functions.
    #
    # You can also add your own functions and call them from here.
    ###################################################################

    ########## OpenLDAP ###############

    ### dump_ldap

    ########## MySQL ##################

    # Dump all grants (permissions), config variables and schema of databases
    ### dump_mysql_meta

    # Dump all databases in a single compressed file
    ### dump_mysql_global
    
    # Dump each database separately, in a compressed file
    ### dump_mysql_per_base
 
    # Dump multiples instances, each in a single compressed file
    ### dump_mysql_instances

    # Dump each table in schema/data files, for all databases
    ### dump_mysql_tabs

    # Run mysqlhotcopy for a specific database (must be configured)
    # dump_mysql_hotcopy

    ########## PostgreSQL #############

    # Dump all databases in a single file (compressed or not)
    ### dump_postgresql_global

    # Dump a specific databse with only some tables, or all but some tables (must be configured)
    ### dump_postgresql_filtered

    # Dump each database separately, in a compressed file
    ### dump_postgresql_per_base

    ########## MongoDB ################
    
    ### dump_mongodb

    ########## Redis ##################

    # Copy data file for all instances
    ### dump_redis

    ########## ElasticSearch ##########

    # Trigger snapshots (must be configured)
    ### dump_elasticsearch_snapshot

    ########## RabbitMQ ###############

    ### dump_rabbitmq

    ########## MegaCli ################

    # Copy RAID config
    ### dump_megacli_config

    ########## Network ################

    # Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls)
    dump_traceroute

    ########## Server state ###########

    # Run dump-server-state to extract system information
    dump_server_state

    # Dump file access control lists
    ### dump_facl

    ###################################################################

    print_error_files_content

    log "STOP LOCAL_TASKS"
}

# Output error files content, if any
print_error_files_content() {
    error_files=$(find "${LOCAL_BACKUP_DIR}/" -type f -name '*.err')
    for error_file in ${error_files}; do
        error_file_size=$(stat -c "%s" "${error_file}")

        # shellcheck disable=SC2086
        if [ ${error_file_size} -gt 0 ]; then
            printf "### cat %s ###\n" "${error_file}" >&2
            cat "${error_file}" >&2
        fi
    done
}
# shellcheck disable=SC2317
mysql_list_databases() {
    port=${1:-"3306"}

    mysql --defaults-extra-file=/etc/mysql/debian.cnf -P "${port}" -e 'show databases' -s --skip-column-names \
        | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)"
}
# shellcheck disable=SC2317
dump_ldap() {
    ## OpenLDAP : example with slapcat
    dump_dir="${LOCAL_BACKUP_DIR}/ldap"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    log "LOCAL_TASKS - start dump_ldap to ${dump_dir}"

    slapcat -n 0 -l "${dump_dir}/config.bak"
    slapcat -n 1 -l "${dump_dir}/data.bak"
    slapcat      -l "${dump_dir}/ldap.bak"

    log "LOCAL_TASKS - stop  dump_ldap"
}
# shellcheck disable=SC2317
dump_mysql_global() {
    dump_dir="${LOCAL_BACKUP_DIR}/mysql-global"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    error_file="${dump_dir}/mysql.bak.err"
    dump_file="${dump_dir}/mysql.bak.gz"
    log "LOCAL_TASKS - start ${dump_file}"

    mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 --opt --all-databases --force --events --hex-blob 2> "${error_file}" \
        | gzip --best > "${dump_file}"

    last_rc=$?
    # shellcheck disable=SC2086
    if [ ${last_rc} -ne 0 ]; then
        error "mysqldump to ${dump_file} returned an error ${last_rc}, check ${error_file}"
        rc=${E_DUMPFAILED}
    else
        rm -f "${error_file}"
    fi
    log "LOCAL_TASKS - stop  ${dump_file}"
}
# shellcheck disable=SC2317
dump_mysql_per_base() {
    dump_dir="${LOCAL_BACKUP_DIR}/mysql-per-base"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    databases=$(mysql_list_databases 3306)
    for database in ${databases}; do
        error_file="${dump_dir}/${database}.err"
        dump_file="${dump_dir}/${database}.sql.gz"
        log "LOCAL_TASKS - start ${dump_file}"

        mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob "${database}" 2> "${error_file}" \
            | gzip --best > "${dump_file}"

        last_rc=$?
        # shellcheck disable=SC2086
        if [ ${last_rc} -ne 0 ]; then
            error "mysqldump to ${dump_file} returned an error ${last_rc}, check ${error_file}"
            rc=${E_DUMPFAILED}
        else
            rm -f "${error_file}"
        fi
        log "LOCAL_TASKS - stop  ${dump_file}"
    done
}
# shellcheck disable=SC2317
dump_mysql_meta() {
    dump_dir="${LOCAL_BACKUP_DIR}/mysql-meta"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    ## Dump all grants (requires 'percona-toolkit' package)
    error_file="${dump_dir}/all_grants.err"
    dump_file="${dump_dir}/all_grants.sql"
    log "LOCAL_TASKS - start ${dump_file}"

    pt-show-grants --flush --no-header 2> "${error_file}" > "${dump_file}"

    last_rc=$?
    # shellcheck disable=SC2086
    if [ ${last_rc} -ne 0 ]; then
        error "pt-show-grants to ${dump_file} returned an error ${last_rc}, check ${error_file}"
        rc=${E_DUMPFAILED}
    else
        rm -f "${error_file}"
    fi
    log "LOCAL_TASKS - stop  ${dump_file}"

    ## Dump all variables
    error_file="${dump_dir}/variables.err"
    dump_file="${dump_dir}/variables.txt"
    log "LOCAL_TASKS - start ${dump_file}"

    mysql -A -e "SHOW GLOBAL VARIABLES;" 2> "${error_file}" > "${dump_file}"

    last_rc=$?
    # shellcheck disable=SC2086
    if [ ${last_rc} -ne 0 ]; then
        error "mysql 'show variables' returned an error ${last_rc}, check ${error_file}"
        rc=${E_DUMPFAILED}
    else
        rm -f "${error_file}"
    fi
    log "LOCAL_TASKS - stop  ${dump_file}"

    ## Schema only (no data) for each databases
    databases=$(mysql_list_databases 3306)
    for database in ${databases}; do
        error_file="${dump_dir}/${database}.schema.err"
        dump_file="${dump_dir}/${database}.schema.sql"
        log "LOCAL_TASKS - start ${dump_file}"

        mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases "${database}" 2> "${error_file}" > "${dump_file}"

        last_rc=$?
        # shellcheck disable=SC2086
        if [ ${last_rc} -ne 0 ]; then
            error "mysqldump to ${dump_file} returned an error ${last_rc}, check ${error_file}"
            rc=${E_DUMPFAILED}
        else
            rm -f "${error_file}"
        fi
        log "LOCAL_TASKS - stop  ${dump_file}"
    done
}
# shellcheck disable=SC2317
dump_mysql_tabs() {
    databases=$(mysql_list_databases 3306)
    for database in ${databases}; do
        dump_dir="${LOCAL_BACKUP_DIR}/mysql-tabs/${database}"
        rm -rf "${dump_dir}"
        # shellcheck disable=SC2174
        mkdir -p -m 700 "${dump_dir}"
        chown -RL mysql "${dump_dir}"

        error_file="${dump_dir}.err"
        log "LOCAL_TASKS - start ${dump_dir}"

        mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments --fields-enclosed-by='\"' --fields-terminated-by=',' -T "${dump_dir}" "${database}" 2> "${error_file}"
 
        last_rc=$?
        # shellcheck disable=SC2086
        if [ ${last_rc} -ne 0 ]; then
            error "mysqldump to ${dump_dir} returned an error ${last_rc}, check ${error_file}"
            rc=${E_DUMPFAILED}
        else
            rm -f "${error_file}"
        fi
        log "LOCAL_TASKS - stop  ${dump_dir}"
    done
}
# shellcheck disable=SC2317
dump_mysql_hotcopy() {
    # customize the list of databases to hot-copy
    databases=""
    for database in ${databases}; do
        dump_dir="${LOCAL_BACKUP_DIR}/mysql-hotcopy/${database}"
        rm -rf "${dump_dir}"
        # shellcheck disable=SC2174
        mkdir -p -m 700 "${dump_dir}"

        error_file="${dump_dir}.err"
        log "LOCAL_TASKS - start ${dump_dir}"

        mysqlhotcopy "${database}" "${dump_dir}/" 2> "${error_file}"

        last_rc=$?
        # shellcheck disable=SC2086
        if [ ${last_rc} -ne 0 ]; then
            error "mysqlhotcopy to ${dump_dir} returned an error ${last_rc}, check ${error_file}"
            rc=${E_DUMPFAILED}
        else
            rm -f "${error_file}"
        fi
        log "LOCAL_TASKS - stop  ${dump_dir}"
    done
}
# shellcheck disable=SC2317
dump_mysql_instances() {
    dump_dir="${LOCAL_BACKUP_DIR}/mysql-instances"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf | cut -d " " -f 3)

    # customize list of instances
    instances=""
    for instance in ${instances}; do
        error_file="${dump_dir}/${instance}.err"
        dump_file="${dump_dir}/${instance}.bak.gz"
        log "LOCAL_TASKS - start ${dump_file}"

        mysqldump -P "${instance}" --opt --all-databases --hex-blob -u mysqladmin -p"${mysqladminpasswd}" 2> "${error_file}" | gzip --best > "${dump_file}"

        last_rc=$?
        # shellcheck disable=SC2086
        if [ ${last_rc} -ne 0 ]; then
            error "mysqldump to ${dump_file} returned an error ${last_rc}, check ${error_file}"
            rc=${E_DUMPFAILED}
        else
            rm -f "${error_file}"
        fi
        log "LOCAL_TASKS - stop  ${dump_file}"
    done
}
# shellcheck disable=SC2317
dump_postgresql_global() {
    dump_dir="${LOCAL_BACKUP_DIR}/postgresql-global"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    ## example with pg_dumpall and with compression
    dump_file="${dump_dir}/pg.dump.bak.gz"
    log "LOCAL_TASKS - start ${dump_file}"

    (sudo -u postgres pg_dumpall) 2> "${error_file}" | gzip --best > "${dump_file}"

    last_rc=$?
    # shellcheck disable=SC2086
    if [ ${last_rc} -ne 0 ]; then
        error "pg_dumpall to ${dump_file} returned an error ${last_rc}, check ${error_file}"
        rc=${E_DUMPFAILED}
    else
        rm -f "${error_file}"
    fi

    log "LOCAL_TASKS - stop  ${dump_file}"

    ## example with pg_dumpall and without compression
    ## WARNING: you need space in ~postgres
    # dump_file="${dump_dir}/pg.dump.bak"
    # log "LOCAL_TASKS - start ${dump_file}"
    # 
    # (su - postgres -c "pg_dumpall > ~/pg.dump.bak") 2> "${error_file}"
    # mv ~postgres/pg.dump.bak "${dump_file}"
    # 
    # log "LOCAL_TASKS - stop  ${dump_file}"
}
# shellcheck disable=SC2317
dump_postgresql_per_base() {
    dump_dir="${LOCAL_BACKUP_DIR}/postgresql-per-base"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    (
        # shellcheck disable=SC2164
        cd /var/lib/postgresql
        databases=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' | grep -v "template.*")
        for database in ${databases} ; do
            error_file="${dump_dir}/${database}.err"
            dump_file="${dump_dir}/${database}.sql.gz"
            log "LOCAL_TASKS - start ${dump_file}"

            (sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d "${database}") 2> "${error_file}" | gzip --best > "${dump_file}"

            last_rc=$?
            # shellcheck disable=SC2086
            if [ ${last_rc} -ne 0 ]; then
                error "pg_dump to ${dump_file} returned an error ${last_rc}, check ${error_file}"
                rc=${E_DUMPFAILED}
            else
                rm -f "${error_file}"
            fi
            log "LOCAL_TASKS - stop  ${dump_file}"
        done
    )
}
# shellcheck disable=SC2317
dump_postgresql_filtered() {
    dump_dir="${LOCAL_BACKUP_DIR}/postgresql-filtered"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    error_file="${dump_dir}/pg-backup.err"
    dump_file="${dump_dir}/pg-backup.tar"
    log "LOCAL_TASKS - start ${dump_file}"

    ## example with all tables from MYBASE excepts TABLE1 and TABLE2
    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f "${dump_file}" -t 'TABLE1' -t 'TABLE2' MYBASE 2> "${error_file}"

    ## example with only TABLE1 and TABLE2 from MYBASE
    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f "${dump_file}" -T 'TABLE1' -T 'TABLE2' MYBASE 2> "${error_file}"

    last_rc=$?
    # shellcheck disable=SC2086
    if [ ${last_rc} -ne 0 ]; then
        error "pg_dump to ${dump_file} returned an error ${last_rc}, check ${error_file}"
        rc=${E_DUMPFAILED}
    else
        rm -f "${error_file}"
    fi
    log "LOCAL_TASKS - stop  ${dump_file}"
}
# shellcheck disable=SC2317
dump_redis() {
    instances=$(find /var/lib/ -mindepth 1 -maxdepth 1 -type d -name 'redis*')
    for instance in ${instances}; do
        name=$(basename "${instance}")
        dump_dir="${LOCAL_BACKUP_DIR}/${name}"
        rm -rf "${dump_dir}"

        if [ -f "${instance}/dump.rdb" ]; then
            # shellcheck disable=SC2174
            mkdir -p -m 700 "${dump_dir}"
            log "LOCAL_TASKS - start ${dump_dir}"

            cp -a "${instance}/dump.rdb" "${dump_dir}/" 2> "${error_file}"

            last_rc=$?
            # shellcheck disable=SC2086
            if [ ${last_rc} -ne 0 ]; then
                error "cp ${instance}/dump.rdb to ${dump_dir} returned an error ${last_rc}, check ${error_file}"
                rc=${E_DUMPFAILED}
            else
                rm -f "${error_file}"
            fi
            log "LOCAL_TASKS - stop  ${dump_dir}"
        fi
    done
}
# shellcheck disable=SC2317
dump_mongodb() {
    ## don't forget to create use with read-only access
    ## > use admin
    ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } )

    dump_dir="${LOCAL_BACKUP_DIR}/mongodump"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    error_file="${dump_dir}.err"
    log "LOCAL_TASKS - start ${dump_dir}"

    mongo_user=""
    mongo_password=""

    mongodump -u "${mongo_user}" -p"${mongo_password}" -o "${dump_dir}/" 2> "${error_file}" > /dev/null

    last_rc=$?
    # shellcheck disable=SC2086
    if [ ${last_rc} -ne 0 ]; then
        error "mongodump to ${dump_dir} returned an error ${last_rc}, check ${error_file}"
        rc=${E_DUMPFAILED}
    else
        rm -f "${error_file}"
    fi
    log "LOCAL_TASKS - stop  ${dump_dir}"
}
# shellcheck disable=SC2317
dump_megacli_config() {
    dump_dir="${LOCAL_BACKUP_DIR}/megacli"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    dump_file="${dump_dir}/megacli.cfg"
    error_file="${dump_dir}/megacli.err"
    log "LOCAL_TASKS - start ${dump_file}"

    megacli -CfgSave -f "${dump_file}" -a0 2> "${error_file}" > /dev/null

    last_rc=$?
    # shellcheck disable=SC2086
    if [ ${last_rc} -ne 0 ]; then
        error "megacli to ${dump_file} returned an error ${last_rc}, check ${error_file}"
        rc=${E_DUMPFAILED}
    else
        rm -f "${error_file}"
    fi
    log "LOCAL_TASKS - stop  ${dump_file}"
}
# shellcheck disable=SC2317
dump_traceroute() {
    dump_dir="${LOCAL_BACKUP_DIR}/traceroute"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    network_targets="8.8.8.8 www.evolix.fr travaux.evolix.net"

    mtr_bin=$(command -v mtr)
    if [ -n "${network_targets}" ] && [ -n "${mtr_bin}" ]; then
        for addr in ${network_targets}; do
            dump_file="${dump_dir}/mtr-${addr}"
            log "LOCAL_TASKS - start ${dump_file}"

            ${mtr_bin} -r "${addr}" > "${dump_file}"

            log "LOCAL_TASKS - stop  ${dump_file}"
        done
    fi

    traceroute_bin=$(command -v traceroute)
    if [ -n "${network_targets}" ] && [ -n "${traceroute_bin}" ]; then
        for addr in ${network_targets}; do
            dump_file="${dump_dir}/traceroute-${addr}"
            log "LOCAL_TASKS - start ${dump_file}"

            ${traceroute_bin} -n "${addr}" > "${dump_file}" 2>&1

            log "LOCAL_TASKS - stop  ${dump_file}"
        done
    fi
}
# shellcheck disable=SC2317
dump_server_state() {
    dump_dir="${LOCAL_BACKUP_DIR}/server-state"
    rm -rf "${dump_dir}"
    # Do not create the directory
    # shellcheck disable=SC2174
    # mkdir -p -m 700 "${dump_dir}"

    log "LOCAL_TASKS - start ${dump_dir}"

    dump_server_state_bin=$(command -v dump-server-state)
    if [ -z "${dump_server_state_bin}" ]; then
        error "dump-server-state is missing"
        rc=1
    else
        if [ "${SYSTEM}" = "linux" ]; then
            ${dump_server_state_bin} --all --dump-dir "${dump_dir}"
            last_rc=$?
            # shellcheck disable=SC2086
            if [ ${last_rc} -ne 0 ]; then
                error "dump-server-state returned an error ${last_rc}, check ${dump_dir}"
                rc=${E_DUMPFAILED}
            fi
        else
            ${dump_server_state_bin} --all --dump-dir "${dump_dir}"
            last_rc=$?
            # shellcheck disable=SC2086
            if [ ${last_rc} -ne 0 ]; then
                error "dump-server-state returned an error ${last_rc}, check ${dump_dir}"
                rc=${E_DUMPFAILED}
            fi
        fi
    fi
    log "LOCAL_TASKS - stop  ${dump_dir}"
}
# shellcheck disable=SC2317
dump_rabbitmq() {
    dump_dir="${LOCAL_BACKUP_DIR}/rabbitmq"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    error_file="${dump_dir}.err"
    dump_file="${dump_dir}/config"
    log "LOCAL_TASKS - start ${dump_file}"

    rabbitmqadmin export "${dump_file}" 2> "${error_file}" >> "${LOGFILE}"

    last_rc=$?
    # shellcheck disable=SC2086
    if [ ${last_rc} -ne 0 ]; then
        error "pg_dump to ${dump_file} returned an error ${last_rc}, check ${error_file}"
        rc=${E_DUMPFAILED}
    else
        rm -f "${error_file}"
    fi
    log "LOCAL_TASKS - stop  ${dump_file}"
}
# shellcheck disable=SC2317
dump_facl() {
    dump_dir="${LOCAL_BACKUP_DIR}/facl"
    rm -rf "${dump_dir}"
    # shellcheck disable=SC2174
    mkdir -p -m 700 "${dump_dir}"

    log "LOCAL_TASKS - start ${dump_dir}"

    getfacl -R /etc  > "${dump_dir}/etc.txt"
    getfacl -R /home > "${dump_dir}/home.txt"
    getfacl -R /usr  > "${dump_dir}/usr.txt"
    getfacl -R /var  > "${dump_dir}/var.txt"

    log "LOCAL_TASKS - stop  ${dump_dir}"
}
# shellcheck disable=SC2317
dump_elasticsearch_snapshot() {
    log "LOCAL_TASKS - start dump_elasticsearch_snapshot"

    ## Take a snapshot as a backup.
    ## Warning: You need to have a path.repo configured.
    ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes

    curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" >> "${LOGFILE}"
    curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" >> "${LOGFILE}"

    # Clustered version here
    # It basically the same thing except that you need to check that NFS is mounted
    # if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2'
    # then
    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" >> "${LOGFILE}"
    #     curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" >> "${LOGFILE}"
    # else
    #     echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
    # fi

    ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
    # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
    # done
    # date=$(/bin/date +%F)
    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" >> "${LOGFILE}"

    log "LOCAL_TASKS - stop  dump_elasticsearch_snapshot"
}

# Call test_server with "HOST:PORT" string
# It will return with 0 if the server is reachable.
# It will return with 1 and a message on stderr if not.
test_server() {
    item=$1
    # split HOST and PORT from the input string
    host=$(echo "${item}" | cut -d':' -f1)
    port=$(echo "${item}" | cut -d':' -f2)

    # Test if the server is accepting connections
    ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" "${host}" -p "${port}" -t "exit"
    # shellcheck disable=SC2181
    if [ $? = 0 ]; then
        # SSH connection is OK
        return 0
    else
        # SSH connection failed
        new_error=$(printf "Failed to connect to \`%s' within %s seconds" "${item}" "${SSH_CONNECT_TIMEOUT}")
        log "${new_error}"
        SERVERS_SSH_ERRORS=$(printf "%s\\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d')

        return 1
    fi
}
# Call pick_server with an optional positive integer to get the nth server in the list.
pick_server() {
    increment=${1:-0}
    list_length=$(echo "${SERVERS}" | wc -w)

    if [ "${increment}" -ge "${list_length}" ]; then
        # We've reached the end of the list
        new_error="No more server available"
        log "${new_error}"
        SERVERS_SSH_ERRORS=$(printf "%s\\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d')

        # Log errors to stderr
        printf "%s\\n" "${SERVERS_SSH_ERRORS}" >&2
        return 1
    fi

    # Extract the day of month, without leading 0 (which would give an octal based number)
    today=$(/bin/date +%e)
    # A salt is useful to randomize the starting point in the list
    # but stay identical each time it's called for a server (based on hostname).
    salt=$(hostname | cksum | cut -d' ' -f1)
    # Pick an integer between 0 and the length of the SERVERS list
    # It changes each day
    item=$(( (today + salt + increment) % list_length ))
    # cut starts counting fields at 1, not 0.
    field=$(( item + 1 ))

    echo "${SERVERS}" | cut -d ' ' -f ${field}
}

sync_tasks() {
    n=0
    server=""
    if [ "${SERVERS_FALLBACK}" = "1" ]; then
        # We try to find a suitable server
        while :; do
            server=$(pick_server "${n}")
            test $? = 0 || exit ${E_NOSRVAVAIL}

            if test_server "${server}"; then
                break
            else
                server=""
                n=$(( n + 1 ))
            fi
        done
    else
        # we force the server
        server=$(pick_server "${n}")
    fi

    SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
    SSH_PORT=$(echo "${server}" | cut -d':' -f2)

    log "START SYNC_TASKS - server=${server}"

    # default paths, depending on system
    if [ "${SYSTEM}" = "linux" ]; then
        declare -a rsync_default_includes=(/bin /boot /lib /opt /sbin /usr)
    else
        declare -a rsync_default_includes=(/bsd /bin /sbin /usr)
    fi
    if [ -f "${CANARY_FILE}" ]; then
        rsync_default_includes+=("${CANARY_FILE}")
    fi

    # reset Rsync log file
    if [ -n "$(command -v truncate)" ]; then 
        truncate -s 0 "${RSYNC_LOGFILE}"
    else
        printf "" > "${RSYNC_LOGFILE}"
    fi

    # Create a temp file for excludes and includes
    rsync_includes_file="$(mktemp --tmpdir "${PROGNAME}.rsync-includes.XXXXXX")"
    rsync_excludes_file="$(mktemp --tmpdir "${PROGNAME}.rsync-excludes.XXXXXX")"
    # … and add them to the list of files to delete at exit
    temp_files+=("${rsync_includes_file}")
    temp_files+=("${rsync_excludes_file}")

    # Store includes/excludes in files
    # without blank lines of comments (# or ;)
    echo "${RSYNC_INCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${rsync_includes_file}"
    echo "${RSYNC_EXCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${rsync_excludes_file}"
    
    # Dump filesystem stats with mtree
    log "SYNC_TASKS - start mtree"

    declare -a mtree_files=()

    # Loop over Rsync includes
    while read -r line ; do
        # … but exclude for mtree what will be excluded by Rsync
        mtree_excludes_file="$(mktemp --tmpdir "${PROGNAME}.mtree-excludes.XXXXXX")"
        temp_files+=("${mtree_excludes_file}")
        grep -E "^([^/]|${line})" "${rsync_excludes_file}" | sed -e "s|^${line}|.|" > "${mtree_excludes_file}"

        mtree_file="/var/log/evobackup.$(basename "${line}").mtree"
        temp_files+=("${mtree_file}")

        mtree -x -c -p "${line}" -X "${mtree_excludes_file}" > "${mtree_file}"
        mtree_files+=("${mtree_file}")
    done < "${rsync_includes_file}"

    if [ "${#mtree_files[@]}" -le 0 ]; then
        error "ERROR: mtree didn't produce any file"
    fi

    log "SYNC_TASKS - stop  mtree (files: ${mtree_files[*]})"


    rsync_bin=$(command -v rsync)
    # Build the final Rsync command

    # Rsync main options
    rsync_main_args=()
    rsync_main_args+=(--archive)
    rsync_main_args+=(--itemize-changes)
    rsync_main_args+=(--quiet)
    rsync_main_args+=(--stats)
    rsync_main_args+=(--human-readable)
    rsync_main_args+=(--relative)
    rsync_main_args+=(--partial)
    rsync_main_args+=(--delete)
    rsync_main_args+=(--delete-excluded)
    rsync_main_args+=(--force)
    rsync_main_args+=(--ignore-errors)
    rsync_main_args+=(--log-file "${RSYNC_LOGFILE}")
    rsync_main_args+=(--rsh "ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'")

    # Rsync excludes
    while read -r line ; do
        rsync_main_args+=(--exclude "${line}")
    done < "${rsync_excludes_file}"

    # Rsync local sources
    # Start with default includes
    # shellcheck disable=SC2206
    rsync_main_args+=(${rsync_default_includes[@]})
    # … and add custom includes
    while read -r line ; do
        rsync_main_args+=("${line}")
    done < "${rsync_includes_file}"

    # Rsync remote destination
    rsync_main_args+=("root@${SSH_SERVER}:/var/backup/")

    # … log it
    log "SYNC_TASKS - Rsync main command : ${rsync_bin} ${rsync_main_args[*]}"

    # … execute it
    ${rsync_bin} "${rsync_main_args[@]}"

    rsync_main_rc=$?

    # Copy last lines of rsync log to the main log
    tail -n 30 "${RSYNC_LOGFILE}" >> "${LOGFILE}"
    # Copy Rsync stats to special file
    tail -n 30 "${RSYNC_LOGFILE}" | grep --invert-match --extended-regexp " [\<\>ch\.\*]\S{10} " > "${RSYNC_STATSFILE}"

    # We ignore rc=24 (vanished files)
    if [ ${rsync_main_rc} -ne 0 ] && [ ${rsync_main_rc} -ne 24 ]; then
        error "rsync returned an error ${rsync_main_rc}, check ${LOGFILE}"
        rc=${E_SYNCFAILED}
    else
        # Build the report Rsync command
        rsync_report_args=()
        # Rsync options
        rsync_report_args+=(--rsh "ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'")
        # Rsync local source
        rsync_report_args+=("${RSYNC_STATSFILE}")
        if [ "${#mtree_files[@]}" -gt 0 ]; then
            rsync_report_args+=("${mtree_files[@]}")
        fi
        # Rsync remote destination
        rsync_report_args+=("root@${SSH_SERVER}:/var/log/")

        # … log it
        log "SYNC_TASKS - Rsync report command : ${rsync_bin} ${rsync_report_args[*]}"

        # … execute it
        ${rsync_bin} "${rsync_report_args[@]}"
    fi

    log "STOP SYNC_TASKS - server=${server}"
}

# Output a message to the log file
log() {
    msg="${1:-$(cat /dev/stdin)}"
    pid=$$
    printf "[%s] %s[%s]: %s\\n" \
        "$(/bin/date +"${DATE_FORMAT}")" "${PROGNAME}" "${pid}" "${msg}" \
        >> "${LOGFILE}"
}
# Output a message to stderr
error() {
    msg="${1:-$(cat /dev/stdin)}"
    pid=$$
    printf "[%s] %s[%s]: %s\\n" \
        "$(/bin/date +"${DATE_FORMAT}")" "${PROGNAME}" "${pid}" "${msg}" \
        >&2
}
# Remove all temporary file created during the execution
# shellcheck disable=SC2317
clean_temp_files() {
    # shellcheck disable=SC2086
    rm -f "${temp_files[@]}"
}
main() {
    START_EPOCH=$(/bin/date +%s)
    log "START GLOBAL - VERSION=${VERSION} LOCAL_TASKS=${LOCAL_TASKS} SYNC_TASKS=${SYNC_TASKS}"

    # shellcheck disable=SC2174
    mkdir -p -m 700 ${LOCAL_BACKUP_DIR}

    ## Force umask
    umask 077

    ## Initialize variable to store SSH connection errors
    SERVERS_SSH_ERRORS=""

    ## Verify other evobackup process and kill if needed
    if [ -e "${PIDFILE}" ]; then
        pid=$(cat "${PIDFILE}")
        # Does process still exist ?
        if kill -0 "${pid}" 2> /dev/null; then
            # Killing the childs of evobackup.
            for ppid in $(pgrep -P "${pid}"); do
                kill -9 "${ppid}";
            done
            # Then kill the main PID.
            kill -9 "${pid}"
            printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\\n" >&2
        else
            rm -f "${PIDFILE}"
        fi
    fi
    echo "$$" > "${PIDFILE}"

    # Initialize a list of files to delete at exit
    # Any file added to the list will also be deleted at exit
    declare -a temp_files=("${PIDFILE}")

    trap clean_temp_files EXIT

    # Update canary to keep track of each run
    update-evobackup-canary --who "${PROGNAME}" --file "${CANARY_FILE}"

    if [ "${LOCAL_TASKS}" = "1" ]; then
        local_tasks
    fi

    if [ "${SYNC_TASKS}" = "1" ]; then
        sync_tasks
    fi

    STOP_EPOCH=$(/bin/date +%s)

    if [ "${SYSTEM}" = "openbsd" ]; then
        start_time=$(/bin/date -f "%s" -j "${START_EPOCH}" +"${DATE_FORMAT}")
        stop_time=$(/bin/date -f "%s" -j "${STOP_EPOCH}" +"${DATE_FORMAT}")
    else
        start_time=$(/bin/date --date="@${START_EPOCH}" +"${DATE_FORMAT}")
        stop_time=$(/bin/date --date="@${STOP_EPOCH}" +"${DATE_FORMAT}")
    fi
    duration=$(( STOP_EPOCH - START_EPOCH ))

    log "STOP GLOBAL - start='${start_time}' stop='${stop_time}' duration=${duration}s"

    tail -20 "${LOGFILE}" | mail -s "[info] EvoBackup - Client ${HOSTNAME}" ${MAIL}
}

# set all programs to C language (english)
export LC_ALL=C

# If expansion is attempted on an unset variable or parameter, the shell prints an
# error message, and, if not interactive, exits with a non-zero status.
set -u
# The pipeline's return status is the value of the last (rightmost) command
# to exit with a non-zero status, or zero if all commands exit successfully.
set -o pipefail

# Default return-code (0 == succes)
rc=0

# Possible error codes
E_NOSRVAVAIL=21 # No server is available
E_SYNCFAILED=20 # Faild sync task
E_DUMPFAILED=10 # Faild dump task

# explicit PATH
PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin

# You can set "linux" or "bsd" manually or let it choose automatically
SYSTEM=$(uname | tr '[:upper:]' '[:lower:]')

# Hostname reported in the mail subject
HOSTNAME=$(hostname)

# Store pid in a file named after this program's name
PROGNAME=$(basename "$0")
PIDFILE="/var/run/${PROGNAME}.pid"

# Customize the log path if you want multiple scripts to have separate log files
LOGFILE="/var/log/evobackup.log"
# Rsync complete log file for the current run
RSYNC_LOGFILE="/var/log/${PROGNAME}.rsync.log"
# Rsync stats for the current run
RSYNC_STATSFILE="/var/log/${PROGNAME}.rsync-stats.log"
# Canary file to update before executing tasks
CANARY_FILE="/zzz_evobackup_canary"

DATE_FORMAT="%Y-%m-%d %H:%M:%S"

# Should we fallback on other servers when the first one is unreachable?
SERVERS_FALLBACK=${SERVERS_FALLBACK:-1}
# timeout (in seconds) for SSH connections
SSH_CONNECT_TIMEOUT=${SSH_CONNECT_TIMEOUT:-90}

# execute main function
main

exit ${rc}