From 2ea9614e3c47a110093ad2cc22465475d4ed6c1e Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 15 Jan 2023 22:56:03 +0100 Subject: [PATCH 01/20] WIP: separate lib and custom code --- client/lib/dump.sh | 529 ++++++++++++++++++++++++++++++++++++++++ client/lib/main.sh | 430 ++++++++++++++++++++++++++++++++ client/lib/utilities.sh | 136 +++++++++++ client/zzz_evobackup.sh | 256 +++++++++++++++++++ 4 files changed, 1351 insertions(+) create mode 100644 client/lib/dump.sh create mode 100644 client/lib/main.sh create mode 100644 client/lib/utilities.sh create mode 100644 client/zzz_evobackup.sh diff --git a/client/lib/dump.sh b/client/lib/dump.sh new file mode 100644 index 0000000..47b8a51 --- /dev/null +++ b/client/lib/dump.sh @@ -0,0 +1,529 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC2317 + +mysql_list_databases() { + port=${1:-"3306"} + + mysql --defaults-extra-file=/etc/mysql/debian.cnf --port="${port}" --execute="show databases" --silent --skip-column-names \ + | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)" +} + +### BEGIN Dump functions #### + +dump_from_lib() { + echo "Dump from lib" +} + +dump_ldap() { + ## OpenLDAP : example with slapcat + local dump_dir="${LOCAL_BACKUP_DIR}/ldap" + rm -rf "${dump_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" + + log "LOCAL_TASKS - start dump_ldap to ${dump_dir}" + + slapcat -n 0 -l "${dump_dir}/config.bak" + slapcat -n 1 -l "${dump_dir}/data.bak" + slapcat -l "${dump_dir}/all.bak" + + log "LOCAL_TASKS - stop dump_ldap" +} +dump_mysql_global() { + local dump_dir="${LOCAL_BACKUP_DIR}/mysql-global" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + local error_file="${errors_dir}/mysql.bak.err" + local dump_file="${dump_dir}/mysql.bak.gz" + log "LOCAL_TASKS - start ${dump_file}" + + mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 --opt --all-databases --force --events --hex-blob 2> "${error_file}" | gzip --best > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_file}" +} +dump_mysql_per_base() { + local dump_dir="${LOCAL_BACKUP_DIR}/mysql-per-base" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + databases=$(mysql_list_databases 3306) + for database in ${databases}; do + local error_file="${errors_dir}/${database}.err" + local dump_file="${dump_dir}/${database}.sql.gz" + log "LOCAL_TASKS - start ${dump_file}" + + mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob "${database}" 2> "${error_file}" | gzip --best > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_file}" + done +} +dump_mysql_meta() { + local dump_dir="${LOCAL_BACKUP_DIR}/mysql-meta" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + ## Dump all grants (requires 'percona-toolkit' package) + local error_file="${errors_dir}/all_grants.err" + local dump_file="${dump_dir}/all_grants.sql" + log "LOCAL_TASKS - start ${dump_file}" + + pt-show-grants --flush --no-header 2> "${error_file}" > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - pt-show-grants to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_file}" + + ## Dump all variables + local error_file="${errors_dir}/variables.err" + local dump_file="${dump_dir}/variables.txt" + log "LOCAL_TASKS - start ${dump_file}" + + mysql -A -e "SHOW GLOBAL VARIABLES;" 2> "${error_file}" > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - mysql 'show variables' returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_file}" + + ## Schema only (no data) for each databases + databases=$(mysql_list_databases 3306) + for database in ${databases}; do + local error_file="${errors_dir}/${database}.schema.err" + local dump_file="${dump_dir}/${database}.schema.sql" + log "LOCAL_TASKS - start ${dump_file}" + + mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases "${database}" 2> "${error_file}" > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_file}" + done +} +dump_mysql_tabs() { + databases=$(mysql_list_databases 3306) + for database in ${databases}; do + local dump_dir="${LOCAL_BACKUP_DIR}/mysql-tabs/${database}" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + chown -RL mysql "${dump_dir}" + + local error_file="${errors_dir}.err" + log "LOCAL_TASKS - start ${dump_dir}" + + mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments --fields-enclosed-by='\"' --fields-terminated-by=',' -T "${dump_dir}" "${database}" 2> "${error_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - mysqldump to ${dump_dir} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_dir}" + done +} +dump_mysql_hotcopy() { + # customize the list of databases to hot-copy + databases="" + for database in ${databases}; do + local dump_dir="${LOCAL_BACKUP_DIR}/mysql-hotcopy/${database}" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + local error_file="${errors_dir}.err" + log "LOCAL_TASKS - start ${dump_dir}" + + mysqlhotcopy "${database}" "${dump_dir}/" 2> "${error_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - mysqlhotcopy to ${dump_dir} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_dir}" + done +} +dump_mysql_instances() { + local dump_dir="${LOCAL_BACKUP_DIR}/mysql-instances" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + mysql_user="mysqladmin" + mysql_passwd=$(grep -m1 'password = .*' /root/.my.cnf | cut -d " " -f 3) + + # customize list of instances + instances="" + for instance in ${instances}; do + local error_file="${errors_dir}/${instance}.err" + local dump_file="${dump_dir}/${instance}.bak.gz" + log "LOCAL_TASKS - start ${dump_file}" + + mysqldump --port="${instance}" --opt --all-databases --hex-blob --user="${mysql_user}" --password="${mysql_passwd}" 2> "${error_file}" | gzip --best > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_file}" + done +} +dump_postgresql_global() { + local dump_dir="${LOCAL_BACKUP_DIR}/postgresql-global" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + ## example with pg_dumpall and with compression + local dump_file="${dump_dir}/pg.dump.bak.gz" + log "LOCAL_TASKS - start ${dump_file}" + + (sudo -u postgres pg_dumpall) 2> "${error_file}" | gzip --best > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - pg_dumpall to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + + log "LOCAL_TASKS - stop ${dump_file}" + + ## example with pg_dumpall and without compression + ## WARNING: you need space in ~postgres + # local dump_file="${dump_dir}/pg.dump.bak" + # log "LOCAL_TASKS - start ${dump_file}" + # + # (su - postgres -c "pg_dumpall > ~/pg.dump.bak") 2> "${error_file}" + # mv ~postgres/pg.dump.bak "${dump_file}" + # + # log "LOCAL_TASKS - stop ${dump_file}" +} +dump_postgresql_per_base() { + local dump_dir="${LOCAL_BACKUP_DIR}/postgresql-per-base" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + ( + # shellcheck disable=SC2164 + cd /var/lib/postgresql + databases=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' | grep -v "template.*") + for database in ${databases} ; do + local error_file="${errors_dir}/${database}.err" + local dump_file="${dump_dir}/${database}.sql.gz" + log "LOCAL_TASKS - start ${dump_file}" + + (sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d "${database}") 2> "${error_file}" | gzip --best > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - pg_dump to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_file}" + done + ) +} +dump_postgresql_filtered() { + local dump_dir="${LOCAL_BACKUP_DIR}/postgresql-filtered" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + local error_file="${errors_dir}/pg-backup.err" + local dump_file="${dump_dir}/pg-backup.tar" + log "LOCAL_TASKS - start ${dump_file}" + + ## example with all tables from MYBASE excepts TABLE1 and TABLE2 + # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f "${dump_file}" -t 'TABLE1' -t 'TABLE2' MYBASE 2> "${error_file}" + + ## example with only TABLE1 and TABLE2 from MYBASE + # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f "${dump_file}" -T 'TABLE1' -T 'TABLE2' MYBASE 2> "${error_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - pg_dump to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_file}" +} +dump_redis() { + instances=$(find /var/lib/ -mindepth 1 -maxdepth 1 -type d -name 'redis*') + for instance in ${instances}; do + name=$(basename "${instance}") + local dump_dir="${LOCAL_BACKUP_DIR}/${name}" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + if [ -f "${instance}/dump.rdb" ]; then + local error_file="${errors_dir}/${instance}.err" + log "LOCAL_TASKS - start ${dump_dir}" + + cp -a "${instance}/dump.rdb" "${dump_dir}/" 2> "${error_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - cp ${instance}/dump.rdb to ${dump_dir} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_dir}" + fi + done +} +dump_mongodb() { + ## don't forget to create use with read-only access + ## > use admin + ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } ) + + local dump_dir="${LOCAL_BACKUP_DIR}/mongodump" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + local error_file="${errors_dir}.err" + log "LOCAL_TASKS - start ${dump_dir}" + + mongo_user="" + mongo_password="" + + mongodump -u "${mongo_user}" -p"${mongo_password}" -o "${dump_dir}/" 2> "${error_file}" > /dev/null + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - mongodump to ${dump_dir} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_dir}" +} +dump_megacli_config() { + local dump_dir="${LOCAL_BACKUP_DIR}/megacli" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + local dump_file="${dump_dir}/megacli.cfg" + local error_file="${errors_dir}/megacli.err" + log "LOCAL_TASKS - start ${dump_file}" + + megacli -CfgSave -f "${dump_file}" -a0 2> "${error_file}" > /dev/null + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - megacli to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_file}" +} +dump_traceroute() { + local dump_dir="${LOCAL_BACKUP_DIR}/traceroute" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + network_targets="8.8.8.8 www.evolix.fr travaux.evolix.net" + + mtr_bin=$(command -v mtr) + if [ -n "${network_targets}" ] && [ -n "${mtr_bin}" ]; then + for addr in ${network_targets}; do + local dump_file="${dump_dir}/mtr-${addr}" + log "LOCAL_TASKS - start ${dump_file}" + + ${mtr_bin} -r "${addr}" > "${dump_file}" + + log "LOCAL_TASKS - stop ${dump_file}" + done + fi + + traceroute_bin=$(command -v traceroute) + if [ -n "${network_targets}" ] && [ -n "${traceroute_bin}" ]; then + for addr in ${network_targets}; do + local dump_file="${dump_dir}/traceroute-${addr}" + log "LOCAL_TASKS - start ${dump_file}" + + ${traceroute_bin} -n "${addr}" > "${dump_file}" 2>&1 + + log "LOCAL_TASKS - stop ${dump_file}" + done + fi +} +dump_server_state() { + local dump_dir="${LOCAL_BACKUP_DIR}/server-state" + rm -rf "${dump_dir}" + # Do not create the directory + # shellcheck disable=SC2174 + # mkdir -p -m 700 "${dump_dir}" + + log "LOCAL_TASKS - start ${dump_dir}" + + dump_server_state_bin=$(command -v dump-server-state) + if [ -z "${dump_server_state_bin}" ]; then + log_error "LOCAL_TASKS - dump-server-state is missing" + rc=1 + else + if [ "${SYSTEM}" = "linux" ]; then + ${dump_server_state_bin} --all --dump-dir "${dump_dir}" + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - dump-server-state returned an error ${last_rc}, check ${dump_dir}" + GLOBAL_RC=${E_DUMPFAILED} + fi + else + ${dump_server_state_bin} --all --dump-dir "${dump_dir}" + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - dump-server-state returned an error ${last_rc}, check ${dump_dir}" + GLOBAL_RC=${E_DUMPFAILED} + fi + fi + fi + log "LOCAL_TASKS - stop ${dump_dir}" +} +dump_rabbitmq() { + local dump_dir="${LOCAL_BACKUP_DIR}/rabbitmq" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + local error_file="${errors_dir}.err" + local dump_file="${dump_dir}/config" + log "LOCAL_TASKS - start ${dump_file}" + + rabbitmqadmin export "${dump_file}" 2> "${error_file}" >> "${LOGFILE}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - pg_dump to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_file}" +} +dump_facl() { + local dump_dir="${LOCAL_BACKUP_DIR}/facl" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + + log "LOCAL_TASKS - start ${dump_dir}" + + getfacl -R /etc > "${dump_dir}/etc.txt" + getfacl -R /home > "${dump_dir}/home.txt" + getfacl -R /usr > "${dump_dir}/usr.txt" + getfacl -R /var > "${dump_dir}/var.txt" + + log "LOCAL_TASKS - stop ${dump_dir}" +} +dump_elasticsearch_snapshot() { + log "LOCAL_TASKS - start dump_elasticsearch_snapshot" + + ## Take a snapshot as a backup. + ## Warning: You need to have a path.repo configured. + ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes + + curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" >> "${LOGFILE}" + curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" >> "${LOGFILE}" + + # Clustered version here + # It basically the same thing except that you need to check that NFS is mounted + # if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2' + # then + # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" >> "${LOGFILE}" + # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" >> "${LOGFILE}" + # else + # echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.' + # fi + + ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by : + # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do + # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}' + # done + # date=$(/bin/date +%F) + # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" >> "${LOGFILE}" + + log "LOCAL_TASKS - stop dump_elasticsearch_snapshot" +} \ No newline at end of file diff --git a/client/lib/main.sh b/client/lib/main.sh new file mode 100644 index 0000000..ddc8070 --- /dev/null +++ b/client/lib/main.sh @@ -0,0 +1,430 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC2317 + +readonly VERSION="23.1-pre" + +# set all programs to C language (english) +export LC_ALL=C + +# If expansion is attempted on an unset variable or parameter, the shell prints an +# error message, and, if not interactive, exits with a non-zero status. +set -u +# The pipeline's return status is the value of the last (rightmost) command +# to exit with a non-zero status, or zero if all commands exit successfully. +set -o pipefail + +local_tasks() { + log_error "The 'local_tasks' function hasn't been customized" +} +# Called from main, it is wrapping the local_tasks function defined in the real script +local_tasks_wrapper() { + log "START LOCAL_TASKS" + + # Remove old log directories + find "${LOCAL_BACKUP_DIR}/" -type d -name "${PROGNAME}.errors-*" -ctime +30 -delete + + # This function must be defined in the calling script + local_tasks + + # TODO: check if this is still needed + # print_error_files_content + + log "STOP LOCAL_TASKS" +} +sync_tasks() { + log_error "The 'sync_tasks' function hasn't been customized" +} +# Called from main, it is wrapping the sync_tasks function defined in the real script +sync_tasks_wrapper() { + declare -a SERVERS # Indexed array for server/port values + declare -a RSYNC_INCLUDES # Indexed array for includes + declare -a RSYNC_EXCLUDES # Indexed array for excludes + + case "${SYSTEM}" in + linux) + declare -a rsync_default_includes=( + /bin + /boot + /lib + /opt + /sbin + /usr + ) + ;; + *bsd) + declare -a rsync_default_includes=( + /bin + /bsd + /sbin + /usr + ) + ;; + *) + echo "Unknown system '${SYSTEM}'" >&2 + exit 1 + ;; + esac + if [ -f "${CANARY_FILE}" ]; then + rsync_default_includes+=("${CANARY_FILE}") + fi + readonly rsync_default_includes + + declare -a rsync_default_excludes=( + /dev + /proc + /run + /sys + /tmp + /usr/doc + /usr/obj + /usr/share/doc + /usr/src + /var/apt + /var/cache + /var/db/munin/*.tmp + /var/lib/amavis/amavisd.sock + /var/lib/amavis/tmp + /var/lib/clamav/*.tmp + /var/lib/elasticsearch + /var/lib/metche + /var/lib/mongodb + /var/lib/munin/*tmp* + /var/lib/mysql + /var/lib/php/sessions + /var/lib/php5 + /var/lib/postgres + /var/lib/postgresql + /var/lib/sympa + /var/lock + /var/run + /var/spool/postfix + /var/spool/smtpd + /var/spool/squid + /var/state + /var/tmp + lost+found + .nfs.* + lxc/*/rootfs/tmp + lxc/*/rootfs/usr/doc + lxc/*/rootfs/usr/obj + lxc/*/rootfs/usr/share/doc + lxc/*/rootfs/usr/src + lxc/*/rootfs/var/apt + lxc/*/rootfs/var/cache + lxc/*/rootfs/var/lib/php5 + lxc/*/rootfs/var/lib/php/sessions + lxc/*/rootfs/var/lock + lxc/*/rootfs/var/run + lxc/*/rootfs/var/state + lxc/*/rootfs/var/tmp + /home/mysqltmp + ) + readonly rsync_default_excludes + + # This function must be defined in the calling script + sync_tasks +} + +sync() { + local sync_name=${1} + local -a rsync_servers=("${!2}") + local -a rsync_includes=("${!3}") + local -a rsync_excludes=("${!4}") + + ## Initialize variable to store SSH connection errors + declare -a SSH_ERRORS=() + + # echo "### sync ###" + + # for server in "${rsync_servers[@]}"; do + # echo "server: ${server}" + # done + + # for include in "${rsync_includes[@]}"; do + # echo "include: ${include}" + # done + + # for exclude in "${rsync_excludes[@]}"; do + # echo "exclude: ${exclude}" + # done + + local -i n=0 + local server="" + if [ "${SERVERS_FALLBACK}" = "1" ]; then + # We try to find a suitable server + while :; do + server=$(pick_server ${n}) + test $? = 0 || exit ${E_NOSRVAVAIL} + + if test_server "${server}"; then + break + else + server="" + n=$(( n + 1 )) + fi + done + else + # we force the server + server=$(pick_server "${n}") + fi + + rsync_server=$(echo "${server}" | cut -d':' -f1) + rsync_port=$(echo "${server}" | cut -d':' -f2) + + log "START SYNC_TASKS - ${sync_name} : server=${server}" + + # Rsync complete log file for the current run + RSYNC_LOGFILE="/var/log/${PROGNAME}.${sync_name}.rsync.log" + # Rsync stats for the current run + RSYNC_STATSFILE="/var/log/${PROGNAME}.${sync_name}.rsync-stats.log" + + # reset Rsync log file + if [ -n "$(command -v truncate)" ]; then + truncate -s 0 "${RSYNC_LOGFILE}" + truncate -s 0 "${RSYNC_STATSFILE}" + else + printf "" > "${RSYNC_LOGFILE}" + printf "" > "${RSYNC_STATSFILE}" + fi + + if [ "${MTREE_ENABLED}" = "1" ]; then + mtree_bin=$(command -v mtree) + + if [ -n "${mtree_bin}" ]; then + # Dump filesystem stats with mtree + log "SYNC_TASKS - start mtree" + + local -a mtree_files=() + + # Loop over Rsync includes + + for i in "${!rsync_includes[@]}"; do + include="${rsync_includes[i]}" + + # … but exclude for mtree what will be excluded by Rsync + mtree_excludes_file="$(mktemp --tmpdir "${PROGNAME}.${sync_name}.mtree-excludes.XXXXXX")" + add_to_temp_files "${mtree_excludes_file}" + + for j in "${!rsync_excludes[@]}"; do + echo "${rsync_excludes[j]}" | grep -E "^([^/]|${include})" | sed -e "s|^${include}|.|" >> "${mtree_excludes_file}" + done + + mtree_file="/var/log/evobackup.$(basename "${include}").mtree" + add_to_temp_files "${mtree_file}" + + ${mtree_bin} -x -c -p "${include}" -X "${mtree_excludes_file}" > "${mtree_file}" + mtree_files+=("${mtree_file}") + done + + if [ "${#mtree_files[@]}" -le 0 ]; then + log_error "SYNC_TASKS - ERROR: mtree didn't produce any file" + fi + + log "SYNC_TASKS - stop mtree (files: ${mtree_files[*]})" + else + log "SYNC_TASKS - skip mtree (missing)" + fi + else + log "SYNC_TASKS - skip mtree (disabled)" + fi + + rsync_bin=$(command -v rsync) + # Build the final Rsync command + + # Rsync main options + rsync_main_args=() + rsync_main_args+=(--archive) + rsync_main_args+=(--itemize-changes) + rsync_main_args+=(--quiet) + rsync_main_args+=(--stats) + rsync_main_args+=(--human-readable) + rsync_main_args+=(--relative) + rsync_main_args+=(--partial) + rsync_main_args+=(--delete) + rsync_main_args+=(--delete-excluded) + rsync_main_args+=(--force) + rsync_main_args+=(--ignore-errors) + rsync_main_args+=(--log-file "${RSYNC_LOGFILE}") + rsync_main_args+=(--rsh "ssh -p ${rsync_port} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'") + + # Rsync excludes + for i in "${!rsync_excludes[@]}"; do + rsync_main_args+=(--exclude "${rsync_excludes[i]}") + done + + # Rsync local sources + rsync_main_args+=("${rsync_includes[@]}") + + # Rsync remote destination + rsync_main_args+=("root@${rsync_server}:${REMOTE_BACKUP_DIR}/") + + # … log it + log "SYNC_TASKS - ${sync_name} Rsync main command : ${rsync_bin} ${rsync_main_args[*]}" + + # … execute it + ${rsync_bin} "${rsync_main_args[@]}" + + rsync_main_rc=$? + + # Copy last lines of rsync log to the main log + tail -n 30 "${RSYNC_LOGFILE}" >> "${LOGFILE}" + # Copy Rsync stats to special file + tail -n 30 "${RSYNC_LOGFILE}" | grep --invert-match --extended-regexp " [\<\>ch\.\*]\S{10} " > "${RSYNC_STATSFILE}" + + # We ignore rc=24 (vanished files) + if [ ${rsync_main_rc} -ne 0 ] && [ ${rsync_main_rc} -ne 24 ]; then + log_error "SYNC_TASKS - ${sync_name} Rsync main command returned an error ${rsync_main_rc}" "${LOGFILE}" + GLOBAL_RC=${E_SYNCFAILED} + else + # Build the report Rsync command + local -a rsync_report_args + + rsync_report_args=() + + # Rsync options + rsync_report_args+=(--rsh "ssh -p ${rsync_port} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'") + + # Rsync local sources + if [ "${#mtree_files[@]}" -gt 0 ]; then + # send mtree files if there is any + rsync_report_args+=("${mtree_files[@]}") + fi + if [ -f "${RSYNC_LOGFILE}" ]; then + # send rsync full log file if it exists + rsync_report_args+=("${RSYNC_LOGFILE}") + fi + if [ -f "${RSYNC_STATSFILE}" ]; then + # send rsync stats log file if it exists + rsync_report_args+=("${RSYNC_STATSFILE}") + fi + + # Rsync remote destination + rsync_report_args+=("root@${rsync_server}:${REMOTE_LOG_DIR}/") + + # … log it + log "SYNC_TASKS - ${sync_name} Rsync report command : ${rsync_bin} ${rsync_report_args[*]}" + + # … execute it + ${rsync_bin} "${rsync_report_args[@]}" + fi + + log "STOP SYNC_TASKS - ${sync_name} server=${server}" +} + +setup() { + # Default return-code (0 == succes) + declare -i GLOBAL_RC=0 + + # Possible error codes + readonly E_NOSRVAVAIL=21 # No server is available + readonly E_SYNCFAILED=20 # Failed sync task + readonly E_DUMPFAILED=10 # Failed dump task + + # explicit PATH + PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin + + # System name (linux, openbsd…) + : "${SYSTEM:=$(uname | tr '[:upper:]' '[:lower:]')}" + + # Hostname (for logs and notifications) + : "${HOSTNAME:=$(hostname)}" + + # Store pid in a file named after this program's name + : "${PROGNAME:=$(basename "$0")}" + : "${PIDFILE:="/var/run/${PROGNAME}.pid"}" + + # Customize the log path if you want multiple scripts to have separate log files + : "${LOGFILE:="/var/log/evobackup.log"}" + + # Canary file to update before executing tasks + : "${CANARY_FILE:="/zzz_evobackup_canary"}" + + # Date format for log messages + : "${DATE_FORMAT:="%Y-%m-%d %H:%M:%S"}" + + # Should we fallback on other servers when the first one is unreachable? + : "${SERVERS_FALLBACK:=1}" + # timeout (in seconds) for SSH connections + : "${SSH_CONNECT_TIMEOUT:=90}" + + : "${LOCAL_BACKUP_DIR:="/home/backup"}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${LOCAL_BACKUP_DIR}" + + : "${ERRORS_DIR:="${LOCAL_BACKUP_DIR}/${PROGNAME}.errors-${START_TIME}"}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${ERRORS_DIR}" + + # Backup directory on remote server + : "${REMOTE_BACKUP_DIR:="/var/backup"}" + # Log directory in remote server + : "${REMOTE_LOG_DIR:="/var/log"}" + + # Email address for notifications + : "${MAIL:="root"}" + + # Email subject for notifications + : "${MAIL_SUBJECT:="[info] EvoBackup - Client ${HOSTNAME}"}" + + # Enable/disable local tasks (default: enabled) + : "${LOCAL_TASKS:=1}" + # Enable/disable sync tasks (default: enabled) + : "${SYNC_TASKS:=1}" + + # Enable/disable mtree (default: enabled) + : "${MTREE_ENABLED:=1}" + + ## Force umask + umask 077 + + # Initialize a list of temporary files + declare -a TEMP_FILES=() + # Any file in this list will be deleted when the program exits + trap "clean_temp_files" EXIT +} + +main() { + # Start timer + START_EPOCH=$(/bin/date +%s) + START_TIME=$(/bin/date +"%Y%m%d%H%M%S") + + # Configure variables and environment + setup + + log "START GLOBAL - VERSION=${VERSION} LOCAL_TASKS=${LOCAL_TASKS} SYNC_TASKS=${SYNC_TASKS}" + + # /!\ Only one backup processus can run at the sametime /!\ + # Based on PID file, kill any running process before continuing + enforce_single_process "${PIDFILE}" + + # Update canary to keep track of each run + update-evobackup-canary --who "${PROGNAME}" --file "${CANARY_FILE}" + + if [ "${LOCAL_TASKS}" = "1" ]; then + local_tasks_wrapper + fi + + if [ "${SYNC_TASKS}" = "1" ]; then + sync_tasks_wrapper + fi + + STOP_EPOCH=$(/bin/date +%s) + + case "${SYSTEM}" in + *bsd) + start_time=$(/bin/date -f "%s" -j "${START_EPOCH}" +"${DATE_FORMAT}") + stop_time=$(/bin/date -f "%s" -j "${STOP_EPOCH}" +"${DATE_FORMAT}") + ;; + *) + start_time=$(/bin/date --date="@${START_EPOCH}" +"${DATE_FORMAT}") + stop_time=$(/bin/date --date="@${STOP_EPOCH}" +"${DATE_FORMAT}") + ;; + esac + duration=$(( STOP_EPOCH - START_EPOCH )) + + log "STOP GLOBAL - start='${start_time}' stop='${stop_time}' duration=${duration}s" + + send_mail + + exit ${GLOBAL_RC} +} \ No newline at end of file diff --git a/client/lib/utilities.sh b/client/lib/utilities.sh new file mode 100644 index 0000000..d17aa3c --- /dev/null +++ b/client/lib/utilities.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# Output a message to the log file +log() { + local msg="${1:-$(cat /dev/stdin)}" + local pid=$$ + + printf "[%s] %s[%s]: %s\\n" \ + "$(/bin/date +"${DATE_FORMAT}")" "${PROGNAME}" "${pid}" "${msg}" \ + >> "${LOGFILE}" +} +log_error() { + local error_msg=${1} + local error_file=${2:""} + + if [ -n "${error_file}" ] && [ -f "${error_file}" ]; then + printf "\n### %s\n" "${error_msg}" >&2 + # shellcheck disable=SC2046 + if [ $(wc -l "${error_file}") -gt 30 ]; then + printf "~~~{%s (tail -30)}\n" "${error_file}" >&2 + tail -n 30 "${error_file}" >&2 + else + printf "~~~{%s}\n" "${error_file}" >&2 + cat "${error_file}" >&2 + fi + printf "~~~\n" >&2 + + log "${error_msg}, check ${error_file}" + else + printf "\n### %s\n" "${error_msg}" >&2 + + log "${error_msg}" + fi + +} +add_to_temp_files() { + TEMP_FILES+=("${1}") +} +# Remove all temporary file created during the execution +clean_temp_files() { + # shellcheck disable=SC2086 + rm -f "${TEMP_FILES[@]}" +} +enforce_single_process() { + local pidfile=$1 + + if [ -e "${pidfile}" ]; then + pid=$(cat "${pidfile}") + # Does process still exist? + if kill -0 "${pid}" 2> /dev/null; then + # Killing the childs of evobackup. + for ppid in $(pgrep -P "${pid}"); do + kill -9 "${ppid}"; + done + # Then kill the main PID. + kill -9 "${pid}" + printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\\n" >&2 + else + rm -f "${pidfile}" + fi + fi + add_to_temp_files "${pidfile}" + + echo "$$" > "${pidfile}" +} + +# Build the error directory (inside ERRORS_DIR) based on the dump directory path +errors_dir_from_dump_dir() { + local dump_dir=$1 + local relative_path=$(realpath --relative-to="${LOCAL_BACKUP_DIR}" "${dump_dir}") + + # return absolute path + realpath --canonicalize-missing "${ERRORS_DIR}/${relative_path}" +} + +# Call test_server with "HOST:PORT" string +# It will return with 0 if the server is reachable. +# It will return with 1 and a message on stderr if not. +test_server() { + local item=$1 + # split HOST and PORT from the input string + local host=$(echo "${item}" | cut -d':' -f1) + local port=$(echo "${item}" | cut -d':' -f2) + + local new_error + + # Test if the server is accepting connections + ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" "${host}" -p "${port}" -t "exit" + # shellcheck disable=SC2181 + if [ $? = 0 ]; then + # SSH connection is OK + return 0 + else + # SSH connection failed + new_error=$(printf "Failed to connect to \`%s' within %s seconds" "${item}" "${SSH_CONNECT_TIMEOUT}") + log "${new_error}" + SSH_ERRORS+=("${new_error}") + + return 1 + fi +} + +# Call pick_server with an optional positive integer to get the nth server in the list. +pick_server() { + local -i increment=${1:-0} + local -i list_length=${#SERVERS[@]} + + if (( increment >= list_length )); then + # We've reached the end of the list + new_error="No more server available" + log "${new_error}" + SSH_ERRORS+=("${new_error}") + + # Log errors to stderr + for i in "${!SSH_ERRORS[@]}"; do + printf "%s\n" "${SSH_ERRORS[i]}" >&2 + done + + return 1 + fi + + # Extract the day of month, without leading 0 (which would give an octal based number) + today=$(/bin/date +%e) + # A salt is useful to randomize the starting point in the list + # but stay identical each time it's called for a server (based on hostname). + salt=$(hostname | cksum | cut -d' ' -f1) + # Pick an integer between 0 and the length of the SERVERS list + # It changes each day + n=$(( (today + salt + increment) % list_length )) + + echo "${SERVERS[n]}" +} + +send_mail() { + tail -20 "${LOGFILE}" | mail -s "${MAIL_SUBJECT}" "${MAIL}" +} \ No newline at end of file diff --git a/client/zzz_evobackup.sh b/client/zzz_evobackup.sh new file mode 100644 index 0000000..537d53c --- /dev/null +++ b/client/zzz_evobackup.sh @@ -0,0 +1,256 @@ +#!/bin/bash +# +# Script Evobackup client +# See https://gitea.evolix.org/evolix/evobackup +# +# Authors: Evolix , +# Gregory Colpart , +# Romain Dessort , +# Benoit Série , +# Tristan Pilat , +# Victor Laborie , +# Jérémy Lecour +# and others. +# +# Licence: AGPLv3 + +source ./lib/utilities.sh +source ./lib/dump.sh +source ./lib/main.sh + +####################################################################### +# +# You must configure the MAIL variable to receive notifications. +# +# There is some optional configuration that you can do +# at the end of this script. +# +# The library (usually installed at /usr/local/lib/evobackup/main.sh) +# also has many variables that you can override for fine-tuning. +# +####################################################################### + +# Email adress for notifications +MAIL=jdoe@example.com + +####################################################################### +# +# The "sync_tasks" function will be called by the main function. +# +# You can customize the variables: +# * "sync_name" (String) +# * "SERVERS" (Array of HOST:PORT) +# * "RSYNC_INCLUDES" (Array of paths to include) +# * "RSYNC_EXCLUDES" (Array of paths to exclude) +# +# The "sync" function can be called multiple times +# with a different set of variables. +# That way you can to sync to various destinations. +# +####################################################################### + +sync_tasks() { + + ########## System-only backup (to Evolix servers) ################# + + # Name your sync task, for logs + sync_name="evolix-system" + + # List of host/port for your sync task + SERVERS=( + node0.backup.evolix.net:2234 + node1.backup.evolix.net:2234 + ) + + # What to include in your sync task + # shellcheck disable=SC2034 + RSYNC_INCLUDES=( + "${rsync_default_includes[@]}" + /etc + /root + /var + ) + + # What to exclude from your sync task + # shellcheck disable=SC2034 + RSYNC_EXCLUDES=( + "${rsync_default_excludes[@]}" + ) + + # Call the sync task + sync "${sync_name}" "SERVERS[@]" "RSYNC_INCLUDES[@]" "RSYNC_EXCLUDES[@]" + + + ########## Full backup (to client servers) ######################## + + # Name your sync task, for logs + sync_name="client-full" + + # List of host/port for your sync task + SERVERS=( + client-backup00.evolix.net:2221 + client-backup01.evolix.net:2221 + ) + + # What to include in your sync task + # shellcheck disable=SC2034 + RSYNC_INCLUDES=( + "${rsync_default_includes[@]}" + /etc + /root + /var + /home + /srv + ) + + # What to exclude from your sync task + # shellcheck disable=SC2034 + RSYNC_EXCLUDES=( + "${rsync_default_excludes[@]}" + /home/foo + ) + + # Call the sync task + sync "${sync_name}" "SERVERS[@]" "RSYNC_INCLUDES[@]" "RSYNC_EXCLUDES[@]" + +} + +####################################################################### +# +# The "local_tasks" function will be called by the main function. +# +# You can call any available "dump_xxx" function +# (usually installed at /usr/local/lib/evobackup/dump.sh) +# +# You can also write some custom functions and call them. +# A "dump_custom" example is available further down. +# +####################################################################### + +local_tasks() { + + ########## OpenLDAP ############### + + ### dump_ldap + + ########## MySQL ################## + + # Dump all grants (permissions), config variables and schema of databases + ### dump_mysql_meta + + # Dump all databases in a single compressed file + ### dump_mysql_global + + # Dump each database separately, in a compressed file + ### dump_mysql_per_base + + # Dump multiples instances, each in a single compressed file + ### dump_mysql_instances + + # Dump each table in schema/data files, for all databases + ### dump_mysql_tabs + + # Run mysqlhotcopy for a specific database (must be configured) + # dump_mysql_hotcopy + + ########## PostgreSQL ############# + + # Dump all databases in a single file (compressed or not) + ### dump_postgresql_global + + # Dump a specific databse with only some tables, or all but some tables (must be configured) + ### dump_postgresql_filtered + + # Dump each database separately, in a compressed file + ### dump_postgresql_per_base + + ########## MongoDB ################ + + ### dump_mongodb + + ########## Redis ################## + + # Copy data file for all instances + ### dump_redis + + ########## ElasticSearch ########## + + # Trigger snapshots (must be configured) + ### dump_elasticsearch_snapshot + + ########## RabbitMQ ############### + + ### dump_rabbitmq + + ########## MegaCli ################ + + # Copy RAID config + ### dump_megacli_config + + ########## Network ################ + + # Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) + ### dump_traceroute + + ########## Server state ########### + + # Run dump-server-state to extract system information + ### dump_server_state + + # Dump file access control lists + ### dump_facl + + # No-op, in case nothing is enabled + : +} + +# This is an example for a custom dump function +# Uncomment, customize and call it from the "local_tasks" function +### dump_custom() { +### # Set dump and errors directories and files +### local dump_dir="${LOCAL_BACKUP_DIR}/custom" +### local dump_file="${dump_dir}/dump.gz" +### local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") +### local error_file="${errors_dir}/dump.err" +### +### # Reset dump and errors directories +### rm -rf "${dump_dir}" "${errors_dir}" +### # shellcheck disable=SC2174 +### mkdir -p -m 700 "${dump_dir}" "${errors_dir}" +### +### # Log the start of the command +### log "LOCAL_TASKS - start ${dump_file}" +### +### # Execute your dump command +### # Send errors to the error file and the data to the dump file +### my-dump-command 2> "${error_file}" > "${dump_file}" +### +### # Check result and deal with potential errors +### local last_rc=$? +### # shellcheck disable=SC2086 +### if [ ${last_rc} -ne 0 ]; then +### log_error "LOCAL_TASKS - my-dump-command to ${dump_file} returned an error ${last_rc}" "${error_file}" +### GLOBAL_RC=${E_DUMPFAILED} +### else +### rm -f "${error_file}" +### fi +### +### # Log the end of the command +### log "LOCAL_TASKS - stop ${dump_file}" +### } + +########## Optional configuration ##################################### + +# If you set a value (like "linux", "openbsd"…) it will be used, +# Default: uname(1) in lowercase. +### SYSTEM="linux" + +# If you set a value it will be used, +# Default: hostname(1). +### HOSTNAME="example-host" + +# Email subect for notifications +### MAIL_SUBJECT="[info] EvoBackup - Client ${HOSTNAME}"}" + +# Call main function +main \ No newline at end of file From 7784ba55481f05b2d6e8b45c347c1e9b77532b41 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 16 Jan 2023 09:58:17 +0100 Subject: [PATCH 02/20] load libraries just before calling main --- client/zzz_evobackup.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/client/zzz_evobackup.sh b/client/zzz_evobackup.sh index 537d53c..83d4944 100644 --- a/client/zzz_evobackup.sh +++ b/client/zzz_evobackup.sh @@ -14,10 +14,6 @@ # # Licence: AGPLv3 -source ./lib/utilities.sh -source ./lib/dump.sh -source ./lib/main.sh - ####################################################################### # # You must configure the MAIL variable to receive notifications. @@ -239,6 +235,12 @@ local_tasks() { ### log "LOCAL_TASKS - stop ${dump_file}" ### } +########## Libraries ################################################## + +source ./lib/utilities.sh +source ./lib/dump.sh +source ./lib/main.sh + ########## Optional configuration ##################################### # If you set a value (like "linux", "openbsd"…) it will be used, From ed7f9e79ae488ff945a2a38d43a4860ca74dea85 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 16 Jan 2023 13:16:19 +0100 Subject: [PATCH 03/20] default value --- client/lib/utilities.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/lib/utilities.sh b/client/lib/utilities.sh index d17aa3c..0f48077 100644 --- a/client/lib/utilities.sh +++ b/client/lib/utilities.sh @@ -11,7 +11,7 @@ log() { } log_error() { local error_msg=${1} - local error_file=${2:""} + local error_file=${2:-""} if [ -n "${error_file}" ] && [ -f "${error_file}" ]; then printf "\n### %s\n" "${error_msg}" >&2 From 2bf4d0dd0f41609f6e1157d6e4a84e9504fdeb36 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 16 Jan 2023 14:25:31 +0100 Subject: [PATCH 04/20] mtree includes must be directories --- client/lib/main.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/client/lib/main.sh b/client/lib/main.sh index ddc8070..090af60 100644 --- a/client/lib/main.sh +++ b/client/lib/main.sh @@ -201,19 +201,21 @@ sync() { for i in "${!rsync_includes[@]}"; do include="${rsync_includes[i]}" - # … but exclude for mtree what will be excluded by Rsync - mtree_excludes_file="$(mktemp --tmpdir "${PROGNAME}.${sync_name}.mtree-excludes.XXXXXX")" - add_to_temp_files "${mtree_excludes_file}" + if [ -d "${include}" ]; then + # … but exclude for mtree what will be excluded by Rsync + mtree_excludes_file="$(mktemp --tmpdir "${PROGNAME}.${sync_name}.mtree-excludes.XXXXXX")" + add_to_temp_files "${mtree_excludes_file}" - for j in "${!rsync_excludes[@]}"; do - echo "${rsync_excludes[j]}" | grep -E "^([^/]|${include})" | sed -e "s|^${include}|.|" >> "${mtree_excludes_file}" - done + for j in "${!rsync_excludes[@]}"; do + echo "${rsync_excludes[j]}" | grep -E "^([^/]|${include})" | sed -e "s|^${include}|.|" >> "${mtree_excludes_file}" + done - mtree_file="/var/log/evobackup.$(basename "${include}").mtree" - add_to_temp_files "${mtree_file}" + mtree_file="/var/log/evobackup.$(basename "${include}").mtree" + add_to_temp_files "${mtree_file}" - ${mtree_bin} -x -c -p "${include}" -X "${mtree_excludes_file}" > "${mtree_file}" - mtree_files+=("${mtree_file}") + ${mtree_bin} -x -c -p "${include}" -X "${mtree_excludes_file}" > "${mtree_file}" + mtree_files+=("${mtree_file}") + fi done if [ "${#mtree_files[@]}" -le 0 ]; then From 0491598c1fe8a5d8d76c7271e2c6045093dd43e8 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 16 Jan 2023 14:26:04 +0100 Subject: [PATCH 05/20] hook functions --- client/lib/main.sh | 39 ++++++++++++++++++++++++++------------- client/zzz_evobackup.sh | 41 ++++++++++++++++++++++++----------------- 2 files changed, 50 insertions(+), 30 deletions(-) diff --git a/client/lib/main.sh b/client/lib/main.sh index 090af60..deb8275 100644 --- a/client/lib/main.sh +++ b/client/lib/main.sh @@ -13,9 +13,9 @@ set -u # to exit with a non-zero status, or zero if all commands exit successfully. set -o pipefail -local_tasks() { - log_error "The 'local_tasks' function hasn't been customized" -} +source "${LIBDIR}/utilities.sh" +source "${LIBDIR}/dump.sh" + # Called from main, it is wrapping the local_tasks function defined in the real script local_tasks_wrapper() { log "START LOCAL_TASKS" @@ -23,17 +23,19 @@ local_tasks_wrapper() { # Remove old log directories find "${LOCAL_BACKUP_DIR}/" -type d -name "${PROGNAME}.errors-*" -ctime +30 -delete - # This function must be defined in the calling script - local_tasks + local_tasks_type="$(type -t local_tasks)" + if [ "${local_tasks_type}" = "function" ]; then + local_tasks + else + log_error "There is no 'local_tasks' function to execute" + fi # TODO: check if this is still needed # print_error_files_content log "STOP LOCAL_TASKS" } -sync_tasks() { - log_error "The 'sync_tasks' function hasn't been customized" -} + # Called from main, it is wrapping the sync_tasks function defined in the real script sync_tasks_wrapper() { declare -a SERVERS # Indexed array for server/port values @@ -121,8 +123,12 @@ sync_tasks_wrapper() { ) readonly rsync_default_excludes - # This function must be defined in the calling script - sync_tasks + sync_tasks_type="$(type -t sync_tasks)" + if [ "${sync_tasks_type}" = "function" ]; then + sync_tasks + else + log_error "There is no 'sync_tasks' function to execute" + fi } sync() { @@ -171,7 +177,7 @@ sync() { rsync_server=$(echo "${server}" | cut -d':' -f1) rsync_port=$(echo "${server}" | cut -d':' -f2) - log "START SYNC_TASKS - ${sync_name} : server=${server}" + log "START SYNC_TASKS - \"${sync_name}\" : server=${server}" # Rsync complete log file for the current run RSYNC_LOGFILE="/var/log/${PROGNAME}.${sync_name}.rsync.log" @@ -261,7 +267,7 @@ sync() { rsync_main_args+=("root@${rsync_server}:${REMOTE_BACKUP_DIR}/") # … log it - log "SYNC_TASKS - ${sync_name} Rsync main command : ${rsync_bin} ${rsync_main_args[*]}" + log "SYNC_TASKS - \"${sync_name}\" Rsync main command : ${rsync_bin} ${rsync_main_args[*]}" # … execute it ${rsync_bin} "${rsync_main_args[@]}" @@ -315,7 +321,7 @@ sync() { setup() { # Default return-code (0 == succes) - declare -i GLOBAL_RC=0 + GLOBAL_RC=0 # Possible error codes readonly E_NOSRVAVAIL=21 # No server is available @@ -376,6 +382,12 @@ setup() { # Enable/disable mtree (default: enabled) : "${MTREE_ENABLED:=1}" + # If "setup_custom" exists and is a function, let's call it + setup_custom_type="$(type -t setup_custom)" + if [ "${setup_custom_type}" = "function" ]; then + setup_custom + fi + ## Force umask umask 077 @@ -385,6 +397,7 @@ setup() { trap "clean_temp_files" EXIT } + main() { # Start timer START_EPOCH=$(/bin/date +%s) diff --git a/client/zzz_evobackup.sh b/client/zzz_evobackup.sh index 83d4944..ba08fa6 100644 --- a/client/zzz_evobackup.sh +++ b/client/zzz_evobackup.sh @@ -186,12 +186,12 @@ local_tasks() { ########## Network ################ # Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) - ### dump_traceroute + dump_traceroute ########## Server state ########### # Run dump-server-state to extract system information - ### dump_server_state + dump_server_state # Dump file access control lists ### dump_facl @@ -235,24 +235,31 @@ local_tasks() { ### log "LOCAL_TASKS - stop ${dump_file}" ### } -########## Libraries ################################################## - -source ./lib/utilities.sh -source ./lib/dump.sh -source ./lib/main.sh - ########## Optional configuration ##################################### -# If you set a value (like "linux", "openbsd"…) it will be used, -# Default: uname(1) in lowercase. -### SYSTEM="linux" +setup_custom() { + # If you set a value (like "linux", "openbsd"…) it will be used, + # Default: uname(1) in lowercase. + ### SYSTEM="linux" -# If you set a value it will be used, -# Default: hostname(1). -### HOSTNAME="example-host" + # If you set a value it will be used, + # Default: hostname(1). + ### HOSTNAME="example-host" -# Email subect for notifications -### MAIL_SUBJECT="[info] EvoBackup - Client ${HOSTNAME}"}" + # Email subect for notifications + ### MAIL_SUBJECT="[info] EvoBackup - Client ${HOSTNAME}" + + # No-op in case nothing is executed + : +} + +########## Libraries ################################################## + +# Change this to wherever you install the libraries +LIBDIR="./lib" + +source "${LIBDIR}/main.sh" + +########## Let's go! ################################################## -# Call main function main \ No newline at end of file From c5d82eda68136f45d03294590a9ec1ab5b592cd0 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 16 Jan 2023 14:26:15 +0100 Subject: [PATCH 06/20] deployment playbook --- client/deploy-evobackup-beta.yml | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 client/deploy-evobackup-beta.yml diff --git a/client/deploy-evobackup-beta.yml b/client/deploy-evobackup-beta.yml new file mode 100644 index 0000000..897a190 --- /dev/null +++ b/client/deploy-evobackup-beta.yml @@ -0,0 +1,33 @@ +--- + +- hosts: lecour-www00 + gather_facts: yes + become: yes + + tasks: + + - name: LIBDIR is present + file: + path: /usr/local/lib/evobackup + state: directory + + - name: libraries are installed + copy: + src: "{{ item }}" + dest: /usr/local/lib/evobackup/ + remote_src: False + owner: root + group: root + mode: "0640" + force: yes + loop: "{{ lookup('fileglob', 'lib/*.sh', wantlist=True) }}" + + - name: script is present + copy: + src: zzz_evobackup.sh + dest: /root/evobackup-beta.sh + remote_src: False + owner: root + group: root + mode: "0750" + force: no \ No newline at end of file From 70fbab9bb04d2e2319cb085d9b9bd0e77418f877 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sat, 28 Jan 2023 16:07:39 +0100 Subject: [PATCH 07/20] Test presence of old config file before trying to delete it --- server/CHANGELOG.md | 2 ++ server/lib/bkctld-remove | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 3012189..a3bfad3 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +* Test presence of old config file before trying to delete it + ### Security ## [22.11] - 2022-11-28 diff --git a/server/lib/bkctld-remove b/server/lib/bkctld-remove index 382bac7..87a0ec0 100755 --- a/server/lib/bkctld-remove +++ b/server/lib/bkctld-remove @@ -49,7 +49,7 @@ fi "${LIBDIR}/bkctld-is-on" "${jail_name}" && "${LIBDIR}/bkctld-stop" "${jail_name}" -rm -f "${CONFDIR}/${jail_name}" +test -f "${CONFDIR}/${jail_name}" && rm -f "${CONFDIR}/${jail_name}" rm -rf "$(jail_config_dir "${jail_name}")" btrfs_bin=$(command -v btrfs) From 767d509390c395ac18c99971b8b05aba2862a60f Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sat, 28 Jan 2023 16:20:36 +0100 Subject: [PATCH 08/20] deploy evobackup beta with configured MAIL and LIBDIR --- client/deploy-evobackup-beta.yml | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/client/deploy-evobackup-beta.yml b/client/deploy-evobackup-beta.yml index 897a190..f17fd47 100644 --- a/client/deploy-evobackup-beta.yml +++ b/client/deploy-evobackup-beta.yml @@ -1,20 +1,23 @@ --- -- hosts: lecour-www00 +- hosts: all gather_facts: yes become: yes - tasks: + vars: + evobackup_mail: alert4@evolix.net + evobackup_libdir: "/usr/local/lib/evobackup" + tasks: - name: LIBDIR is present file: - path: /usr/local/lib/evobackup + path: "{{ evobackup_libdir }}" state: directory - name: libraries are installed copy: src: "{{ item }}" - dest: /usr/local/lib/evobackup/ + dest: "{{ evobackup_libdir }}/" remote_src: False owner: root group: root @@ -25,9 +28,21 @@ - name: script is present copy: src: zzz_evobackup.sh - dest: /root/evobackup-beta.sh + dest: /etc/cron.daily/zzz_evobackup remote_src: False owner: root group: root mode: "0750" - force: no \ No newline at end of file + force: no + + - name: Email is customized + replace: + dest: /etc/cron.daily/zzz_evobackup + regexp: "^MAIL=.*" + replace: "MAIL={{ evobackup_mail }}" + + - name: LIBDIR is customized + replace: + dest: /etc/cron.daily/zzz_evobackup + regexp: "^LIBDIR=.*" + replace: "LIBDIR=\"{{ evobackup_libdir }}\"" \ No newline at end of file From d532ac83da708daf55b1318f821f33aebd21eae4 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sat, 28 Jan 2023 16:20:51 +0100 Subject: [PATCH 09/20] client: declare variable earlier --- client/lib/main.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/lib/main.sh b/client/lib/main.sh index deb8275..b8723d0 100644 --- a/client/lib/main.sh +++ b/client/lib/main.sh @@ -193,6 +193,9 @@ sync() { printf "" > "${RSYNC_STATSFILE}" fi + # Initialize variable here, we need it later + local -a mtree_files=() + if [ "${MTREE_ENABLED}" = "1" ]; then mtree_bin=$(command -v mtree) @@ -200,10 +203,7 @@ sync() { # Dump filesystem stats with mtree log "SYNC_TASKS - start mtree" - local -a mtree_files=() - # Loop over Rsync includes - for i in "${!rsync_includes[@]}"; do include="${rsync_includes[i]}" From 149b5d0e8d62cad6e8040857ee4219a02a38cd2b Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sat, 28 Jan 2023 21:14:22 +0100 Subject: [PATCH 10/20] comments --- client/zzz_evobackup.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/client/zzz_evobackup.sh b/client/zzz_evobackup.sh index ba08fa6..3653d18 100644 --- a/client/zzz_evobackup.sh +++ b/client/zzz_evobackup.sh @@ -53,12 +53,14 @@ sync_tasks() { sync_name="evolix-system" # List of host/port for your sync task + # shellcheck disable=SC2034 SERVERS=( node0.backup.evolix.net:2234 node1.backup.evolix.net:2234 ) # What to include in your sync task + # Add or remove paths if you need # shellcheck disable=SC2034 RSYNC_INCLUDES=( "${rsync_default_includes[@]}" @@ -68,6 +70,7 @@ sync_tasks() { ) # What to exclude from your sync task + # Add or remove paths if you need # shellcheck disable=SC2034 RSYNC_EXCLUDES=( "${rsync_default_excludes[@]}" @@ -83,12 +86,14 @@ sync_tasks() { sync_name="client-full" # List of host/port for your sync task + # shellcheck disable=SC2034 SERVERS=( client-backup00.evolix.net:2221 client-backup01.evolix.net:2221 ) # What to include in your sync task + # Add or remove paths if you need # shellcheck disable=SC2034 RSYNC_INCLUDES=( "${rsync_default_includes[@]}" @@ -100,10 +105,10 @@ sync_tasks() { ) # What to exclude from your sync task + # Add or remove paths if you need # shellcheck disable=SC2034 RSYNC_EXCLUDES=( "${rsync_default_excludes[@]}" - /home/foo ) # Call the sync task From 2e9eb4a946aa883b7624fb2b8d0e510426dd97a7 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 8 Feb 2023 22:51:38 +0100 Subject: [PATCH 11/20] variable for script path --- client/deploy-evobackup-beta.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/deploy-evobackup-beta.yml b/client/deploy-evobackup-beta.yml index f17fd47..a00df7c 100644 --- a/client/deploy-evobackup-beta.yml +++ b/client/deploy-evobackup-beta.yml @@ -5,6 +5,7 @@ become: yes vars: + evobackup_script_path: /etc/cron.daily/zzz_evobackup_beta evobackup_mail: alert4@evolix.net evobackup_libdir: "/usr/local/lib/evobackup" @@ -28,7 +29,7 @@ - name: script is present copy: src: zzz_evobackup.sh - dest: /etc/cron.daily/zzz_evobackup + dest: "{{ evobackup_script_path }}" remote_src: False owner: root group: root From 50f81f2716094149d68c477d4d791a2e3b7934e1 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 8 Feb 2023 22:53:28 +0100 Subject: [PATCH 12/20] Add options for dump functions --- client/lib/dump.sh | 1109 ++++++++++++++++++++++++++++++++++++--- client/zzz_evobackup.sh | 28 +- 2 files changed, 1038 insertions(+), 99 deletions(-) diff --git a/client/lib/dump.sh b/client/lib/dump.sh index 47b8a51..9f1f3c2 100644 --- a/client/lib/dump.sh +++ b/client/lib/dump.sh @@ -10,10 +10,11 @@ mysql_list_databases() { ### BEGIN Dump functions #### -dump_from_lib() { - echo "Dump from lib" -} - +####################################################################### +# Dump LDAP files (config, data, all) +# +# Arguments: +####################################################################### dump_ldap() { ## OpenLDAP : example with slapcat local dump_dir="${LOCAL_BACKUP_DIR}/ldap" @@ -29,6 +30,14 @@ dump_ldap() { log "LOCAL_TASKS - stop dump_ldap" } + +####################################################################### +# Dump a single compressed file of all databases of an instance +# +# Arguments: +# --masterdata (default: ) +# --port=[Integer] (default: 3306) +####################################################################### dump_mysql_global() { local dump_dir="${LOCAL_BACKUP_DIR}/mysql-global" local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") @@ -40,7 +49,62 @@ dump_mysql_global() { local dump_file="${dump_dir}/mysql.bak.gz" log "LOCAL_TASKS - start ${dump_file}" - mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 --opt --all-databases --force --events --hex-blob 2> "${error_file}" | gzip --best > "${dump_file}" + local option_masterdata="" + local option_port="3306" + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case $1 in + --masterdata) + option_masterdata=1 + ;; + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - unkwnown option (ignored): '${1}'" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + declare -a options + options+=("${option_masterdata}") + options+=(--defaults-extra-file=/etc/mysql/debian.cnf) + options+=(--port="${option_port}") + options+=(--opt) + options+=(--all-databases) + options+=(--force) + options+=(--events) + options+=(--hex-blob) + + mysqldump "${options[@]}" 2> "${error_file}" | gzip --best > "${dump_file}" local last_rc=$? # shellcheck disable=SC2086 @@ -52,6 +116,13 @@ dump_mysql_global() { fi log "LOCAL_TASKS - stop ${dump_file}" } + +####################################################################### +# Dump a compressed file per database of an instance +# +# Arguments: +# --port=[Integer] (default: 3306) +####################################################################### dump_mysql_per_base() { local dump_dir="${LOCAL_BACKUP_DIR}/mysql-per-base" local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") @@ -59,13 +130,61 @@ dump_mysql_per_base() { # shellcheck disable=SC2174 mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - databases=$(mysql_list_databases 3306) + local option_port="3306" + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case $1 in + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - unkwnown option (ignored): '${1}'" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + declare -a options + options+=(--defaults-extra-file=/etc/mysql/debian.cnf) + options+=(--port="${option_port}") + options+=(--force) + options+=(--events) + options+=(--hex-blob) + + databases=$(mysql_list_databases ${option_port}) for database in ${databases}; do local error_file="${errors_dir}/${database}.err" local dump_file="${dump_dir}/${database}.sql.gz" log "LOCAL_TASKS - start ${dump_file}" - mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob "${database}" 2> "${error_file}" | gzip --best > "${dump_file}" + mysqldump "${options[@]}" "${database}" 2> "${error_file}" | gzip --best > "${dump_file}" local last_rc=$? # shellcheck disable=SC2086 @@ -78,6 +197,13 @@ dump_mysql_per_base() { log "LOCAL_TASKS - stop ${dump_file}" done } + +####################################################################### +# Dump grants, variables and databases schemas for an instance +# +# Arguments: +# --port=[Integer] (default: 3306) +####################################################################### dump_mysql_meta() { local dump_dir="${LOCAL_BACKUP_DIR}/mysql-meta" local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") @@ -85,12 +211,58 @@ dump_mysql_meta() { # shellcheck disable=SC2174 mkdir -p -m 700 "${dump_dir}" "${errors_dir}" + local option_port="3306" + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case $1 in + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - unkwnown option (ignored): '${1}'" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + ## Dump all grants (requires 'percona-toolkit' package) local error_file="${errors_dir}/all_grants.err" local dump_file="${dump_dir}/all_grants.sql" log "LOCAL_TASKS - start ${dump_file}" - pt-show-grants --flush --no-header 2> "${error_file}" > "${dump_file}" + declare -a options + options+=(--port "${option_port}") + options+=(--flush) + options+=(--no-header) + + pt-show-grants "${options[@]}" 2> "${error_file}" > "${dump_file}" local last_rc=$? # shellcheck disable=SC2086 @@ -107,7 +279,12 @@ dump_mysql_meta() { local dump_file="${dump_dir}/variables.txt" log "LOCAL_TASKS - start ${dump_file}" - mysql -A -e "SHOW GLOBAL VARIABLES;" 2> "${error_file}" > "${dump_file}" + declare -a options + options+=(--port="${option_port}") + options+=(-A) + options+=(-e "SHOW GLOBAL VARIABLES;") + + mysql "${options[@]}" 2> "${error_file}" > "${dump_file}" local last_rc=$? # shellcheck disable=SC2086 @@ -120,13 +297,20 @@ dump_mysql_meta() { log "LOCAL_TASKS - stop ${dump_file}" ## Schema only (no data) for each databases - databases=$(mysql_list_databases 3306) + databases=$(mysql_list_databases "${option_port}") for database in ${databases}; do local error_file="${errors_dir}/${database}.schema.err" local dump_file="${dump_dir}/${database}.schema.sql" log "LOCAL_TASKS - start ${dump_file}" - mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases "${database}" 2> "${error_file}" > "${dump_file}" + declare -a options + options+=(--defaults-extra-file=/etc/mysql/debian.cnf) + options+=(--port="${option_port}") + options+=(--force) + options+=(--no-data) + options+=(--databases "${database}") + + mysqldump "${options[@]}" 2> "${error_file}" > "${dump_file}" local last_rc=$? # shellcheck disable=SC2086 @@ -139,6 +323,13 @@ dump_mysql_meta() { log "LOCAL_TASKS - stop ${dump_file}" done } + +####################################################################### +# Dump "tabs style" separate schema/data for each database of an instance +# +# Arguments: +# --port=[Integer] (default: 3306) +####################################################################### dump_mysql_tabs() { databases=$(mysql_list_databases 3306) for database in ${databases}; do @@ -152,7 +343,63 @@ dump_mysql_tabs() { local error_file="${errors_dir}.err" log "LOCAL_TASKS - start ${dump_dir}" - mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments --fields-enclosed-by='\"' --fields-terminated-by=',' -T "${dump_dir}" "${database}" 2> "${error_file}" + local option_port="3306" + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case $1 in + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - unkwnown option (ignored): '${1}'" + printf 'WARN: Unknown option (ignored): %s\n' "$1" >&2 + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + declare -a options + options+=(--defaults-extra-file=/etc/mysql/debian.cnf) + options+=(--port="${option_port}") + options+=(--force) + options+=(--quote-names) + options+=(--opt) + options+=(--events) + options+=(--hex-blob) + options+=(--skip-comments) + options+=(--fields-enclosed-by='\"') + options+=(--fields-terminated-by=',') + options+=(--tab="${dump_dir}") + options+=("${database}") + + mysqldump "${options[@]}" 2> "${error_file}" local last_rc=$? # shellcheck disable=SC2086 @@ -165,62 +412,136 @@ dump_mysql_tabs() { log "LOCAL_TASKS - stop ${dump_dir}" done } -dump_mysql_hotcopy() { - # customize the list of databases to hot-copy - databases="" - for database in ${databases}; do - local dump_dir="${LOCAL_BACKUP_DIR}/mysql-hotcopy/${database}" - local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - local error_file="${errors_dir}.err" - log "LOCAL_TASKS - start ${dump_dir}" - - mysqlhotcopy "${database}" "${dump_dir}/" 2> "${error_file}" - - local last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - mysqlhotcopy to ${dump_dir} returned an error ${last_rc}" "${error_file}" - GLOBAL_RC=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_dir}" - done -} -dump_mysql_instances() { +####################################################################### +# Dump a single file for all databases of an instance +# using a custom authentication, instead of /etc/mysql/debian.cnf +# +# Arguments: +# --port=[Integer] (default: 3306) +# --user=[String] (default: ) +# --password=[String] (default: ) +####################################################################### +dump_mysql_instance() { local dump_dir="${LOCAL_BACKUP_DIR}/mysql-instances" local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") rm -rf "${dump_dir}" "${errors_dir}" # shellcheck disable=SC2174 mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - mysql_user="mysqladmin" - mysql_passwd=$(grep -m1 'password = .*' /root/.my.cnf | cut -d " " -f 3) + local option_port="" + local option_user="" + local option_password="" + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case $1 in + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + ;; + --user) + # user options, with value separated by space + if [ -n "$2" ]; then + option_user="${2}" + shift + else + log_error "LOCAL_TASKS - '--user' requires a non-empty option argument." + exit 1 + fi + ;; + --user=?*) + # user options, with value separated by = + option_user="${1#*=}" + ;; + --user=) + # user options, without value + log_error "LOCAL_TASKS - '--user' requires a non-empty option argument." + exit 1 + ;; + --password) + # password options, with value separated by space + if [ -n "$2" ]; then + option_password="${2}" + shift + else + log_error "LOCAL_TASKS - '--password' requires a non-empty option argument." + exit 1 + fi + ;; + --password=?*) + # password options, with value separated by = + option_password="${1#*=}" + ;; + --password=) + # password options, without value + log_error "LOCAL_TASKS - '--password' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - unkwnown option (ignored): '${1}'" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac - # customize list of instances - instances="" - for instance in ${instances}; do - local error_file="${errors_dir}/${instance}.err" - local dump_file="${dump_dir}/${instance}.bak.gz" - log "LOCAL_TASKS - start ${dump_file}" - - mysqldump --port="${instance}" --opt --all-databases --hex-blob --user="${mysql_user}" --password="${mysql_passwd}" 2> "${error_file}" | gzip --best > "${dump_file}" - - local last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}" - GLOBAL_RC=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_file}" + shift done + + declare -a options + options+=(--port="${option_port}") + options+=(--user="${option_user}") + options+=(--password="${option_password}") + options+=(--force) + options+=(--opt) + options+=(--all-databases) + options+=(--events) + options+=(--hex-blob) + + local error_file="${errors_dir}/${option_port}.err" + local dump_file="${dump_dir}/${option_port}.bak.gz" + log "LOCAL_TASKS - start ${dump_file}" + + mysqldump "${options[@]}" 2> "${error_file}" | gzip --best > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${dump_file}" } + +####################################################################### +# Dump a single file of all PostgreSQL databases +# +# Arguments: +####################################################################### dump_postgresql_global() { local dump_dir="${LOCAL_BACKUP_DIR}/postgresql-global" local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") @@ -255,6 +576,12 @@ dump_postgresql_global() { # # log "LOCAL_TASKS - stop ${dump_file}" } + +####################################################################### +# Dump a compressed file per database +# +# Arguments: +####################################################################### dump_postgresql_per_base() { local dump_dir="${LOCAL_BACKUP_DIR}/postgresql-per-base" local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") @@ -285,6 +612,14 @@ dump_postgresql_per_base() { done ) } + +####################################################################### +# Dump a compressed file per database +# +# Arguments: +# +# TODO: add arguments to include/exclude tables +####################################################################### dump_postgresql_filtered() { local dump_dir="${LOCAL_BACKUP_DIR}/postgresql-filtered" local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") @@ -312,9 +647,66 @@ dump_postgresql_filtered() { fi log "LOCAL_TASKS - stop ${dump_file}" } + +####################################################################### +# Copy dump file of Redis instances +# +# Arguments: +# --instances=[Integer] (default: all) +####################################################################### dump_redis() { - instances=$(find /var/lib/ -mindepth 1 -maxdepth 1 -type d -name 'redis*') - for instance in ${instances}; do + all_instances=$(find /var/lib/ -mindepth 1 -maxdepth 1 -type d -name 'redis*') + + local option_instances="" + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case $1 in + --instances) + # instances options, with key and value separated by space + if [ -n "$2" ]; then + if [ "${2}" == "all" ]; then + read -a option_instances <<< "${all_instances}" + else + IFS="," read -a option_instances <<< "${2}" + fi + shift + else + log_error "LOCAL_TASKS - '--instances' requires a non-empty option argument." + exit 1 + fi + ;; + --instances=?*) + # instances options, with key and value separated by = + if [ "${1#*=}" == "all" ]; then + read -a option_instances <<< "${all_instances}" + else + IFS="," read -a option_instances <<< "${1#*=}" + fi + ;; + --instances=) + # instances options, without value + log_error "LOCAL_TASKS - '--instances' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - unkwnown option (ignored): '${1}'" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + for instance in "${option_instances[@]}"; do name=$(basename "${instance}") local dump_dir="${LOCAL_BACKUP_DIR}/${name}" local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") @@ -337,9 +729,20 @@ dump_redis() { rm -f "${error_file}" fi log "LOCAL_TASKS - stop ${dump_dir}" + else + log_error "LOCAL_TASKS - '${instance}/dump.rdb' not found." fi done } + +####################################################################### +# Dump all collections of a MongoDB database +# using a custom authentication, instead of /etc/mysql/debian.cnf +# +# Arguments: +# --user=[String] (default: ) +# --password=[String] (default: ) +####################################################################### dump_mongodb() { ## don't forget to create use with read-only access ## > use admin @@ -354,10 +757,73 @@ dump_mongodb() { local error_file="${errors_dir}.err" log "LOCAL_TASKS - start ${dump_dir}" - mongo_user="" - mongo_password="" + local option_user="" + local option_password="" + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case $1 in + --user) + # user options, with value separated by space + if [ -n "$2" ]; then + option_user="${2}" + shift + else + log_error "LOCAL_TASKS - '--user' requires a non-empty option argument." + exit 1 + fi + ;; + --user=?*) + # user options, with value separated by = + option_user="${1#*=}" + ;; + --user=) + # user options, without value + log_error "LOCAL_TASKS - '--user' requires a non-empty option argument." + exit 1 + ;; + --password) + # password options, with value separated by space + if [ -n "$2" ]; then + option_password="${2}" + shift + else + log_error "LOCAL_TASKS - '--password' requires a non-empty option argument." + exit 1 + fi + ;; + --password=?*) + # password options, with value separated by = + option_password="${1#*=}" + ;; + --password=) + # password options, without value + log_error "LOCAL_TASKS - '--password' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - unkwnown option (ignored): '${1}'" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac - mongodump -u "${mongo_user}" -p"${mongo_password}" -o "${dump_dir}/" 2> "${error_file}" > /dev/null + shift + done + + declare -a options + options+=(--username="${option_user}") + options+=(--password="${option_password}") + options+=(--out="${dump_dir}/") + + mongodump "${options[@]}" 2> "${error_file}" > /dev/null local last_rc=$? # shellcheck disable=SC2086 @@ -369,6 +835,12 @@ dump_mongodb() { fi log "LOCAL_TASKS - stop ${dump_dir}" } + +####################################################################### +# Dump MegaCLI configuration +# +# Arguments: +####################################################################### dump_megacli_config() { local dump_dir="${LOCAL_BACKUP_DIR}/megacli" local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") @@ -392,6 +864,13 @@ dump_megacli_config() { fi log "LOCAL_TASKS - stop ${dump_file}" } + +####################################################################### +# Save some traceroute/mtr results +# +# Arguments: +# --targets=[IP,HOST] (default: ) +####################################################################### dump_traceroute() { local dump_dir="${LOCAL_BACKUP_DIR}/traceroute" local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") @@ -399,32 +878,80 @@ dump_traceroute() { # shellcheck disable=SC2174 mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - network_targets="8.8.8.8 www.evolix.fr travaux.evolix.net" + local option_targets="" + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case $1 in + --targets) + # targets options, with key and value separated by space + if [ -n "$2" ]; then + IFS="," read -a option_targets <<< "${2}" + shift + else + log_error "LOCAL_TASKS - '--targets' requires a non-empty option argument." + exit 1 + fi + ;; + --targets=?*) + # targets options, with key and value separated by = + IFS="," read -a option_targets <<< "${1#*=}" + ;; + --targets=) + # targets options, without value + log_error "LOCAL_TASKS - '--targets' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - unkwnown option (ignored): '${1}'" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done mtr_bin=$(command -v mtr) - if [ -n "${network_targets}" ] && [ -n "${mtr_bin}" ]; then - for addr in ${network_targets}; do - local dump_file="${dump_dir}/mtr-${addr}" + if [ -n "${mtr_bin}" ]; then + for target in "${option_targets[@]}"; do + local dump_file="${dump_dir}/mtr-${target}" log "LOCAL_TASKS - start ${dump_file}" - ${mtr_bin} -r "${addr}" > "${dump_file}" + ${mtr_bin} -r "${target}" > "${dump_file}" log "LOCAL_TASKS - stop ${dump_file}" done fi traceroute_bin=$(command -v traceroute) - if [ -n "${network_targets}" ] && [ -n "${traceroute_bin}" ]; then - for addr in ${network_targets}; do - local dump_file="${dump_dir}/traceroute-${addr}" + if [ -n "${traceroute_bin}" ]; then + for target in "${option_targets[@]}"; do + local dump_file="${dump_dir}/traceroute-${target}" log "LOCAL_TASKS - start ${dump_file}" - ${traceroute_bin} -n "${addr}" > "${dump_file}" 2>&1 + ${traceroute_bin} -n "${target}" > "${dump_file}" 2>&1 log "LOCAL_TASKS - stop ${dump_file}" done fi } + +####################################################################### +# Save many system information, using dump_server_state +# +# Arguments: +# +# TODO: pass arguments to the dump_server_state command +# with defaults and overrides (incuding "dump-dir") +####################################################################### dump_server_state() { local dump_dir="${LOCAL_BACKUP_DIR}/server-state" rm -rf "${dump_dir}" @@ -459,6 +986,14 @@ dump_server_state() { fi log "LOCAL_TASKS - stop ${dump_dir}" } + +####################################################################### +# Save RabbitMQ data +# +# Arguments: +# +# Warning: This has been poorly tested +####################################################################### dump_rabbitmq() { local dump_dir="${LOCAL_BACKUP_DIR}/rabbitmq" local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") @@ -482,6 +1017,12 @@ dump_rabbitmq() { fi log "LOCAL_TASKS - stop ${dump_file}" } + +####################################################################### +# Save Files ACL on various partitions. +# +# Arguments: +####################################################################### dump_facl() { local dump_dir="${LOCAL_BACKUP_DIR}/facl" local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") @@ -498,32 +1039,430 @@ dump_facl() { log "LOCAL_TASKS - stop ${dump_dir}" } -dump_elasticsearch_snapshot() { - log "LOCAL_TASKS - start dump_elasticsearch_snapshot" + +####################################################################### +# Snapshot Elasticsearch data (single-node cluster) +# +# Arguments: +# --protocol=[String] (default: http) +# --host=[String] (default: localhost) +# --port=[Integer] (default: 9200) +# --user=[String] (default: ) +# --password=[String] (default: ) +# --repository=[String] (default: snaprepo) +# --snapshot=[String] (default: snapshot.daily) +####################################################################### +dump_elasticsearch_snapshot_singlenode() { + log "LOCAL_TASKS - start dump_elasticsearch_snapshot_singlenode" + + local option_protocol="http" + local option_host="localhost" + local option_port="9200" + local option_user="" + local option_password="" + local option_repository="snaprepo" + local option_snapshot="snapshot.daily" + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case $1 in + --protocol) + # protocol options, with value separated by space + if [ -n "$2" ]; then + option_protocol="${2}" + shift + else + log_error "LOCAL_TASKS - '--protocol' requires a non-empty option argument." + exit 1 + fi + ;; + --protocol=?*) + # protocol options, with value separated by = + option_protocol="${1#*=}" + ;; + --protocol=) + # protocol options, without value + log_error "LOCAL_TASKS - '--protocol' requires a non-empty option argument." + exit 1 + ;; + --host) + # host options, with value separated by space + if [ -n "$2" ]; then + option_host="${2}" + shift + else + log_error "LOCAL_TASKS - '--host' requires a non-empty option argument." + exit 1 + fi + ;; + --host=?*) + # host options, with value separated by = + option_host="${1#*=}" + ;; + --host=) + # host options, without value + log_error "LOCAL_TASKS - '--host' requires a non-empty option argument." + exit 1 + ;; + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + ;; + --user) + # user options, with value separated by space + if [ -n "$2" ]; then + option_user="${2}" + shift + else + log_error "LOCAL_TASKS - '--user' requires a non-empty option argument." + exit 1 + fi + ;; + --user=?*) + # user options, with value separated by = + option_user="${1#*=}" + ;; + --user=) + # user options, without value + log_error "LOCAL_TASKS - '--user' requires a non-empty option argument." + exit 1 + ;; + --password) + # password options, with value separated by space + if [ -n "$2" ]; then + option_password="${2}" + shift + else + log_error "LOCAL_TASKS - '--password' requires a non-empty option argument." + exit 1 + fi + ;; + --password=?*) + # password options, with value separated by = + option_password="${1#*=}" + ;; + --password=) + # password options, without value + log_error "LOCAL_TASKS - '--password' requires a non-empty option argument." + exit 1 + ;; + --repository) + # repository options, with value separated by space + if [ -n "$2" ]; then + option_repository="${2}" + shift + else + log_error "LOCAL_TASKS - '--repository' requires a non-empty option argument." + exit 1 + fi + ;; + --repository=?*) + # repository options, with value separated by = + option_repository="${1#*=}" + ;; + --repository=) + # repository options, without value + log_error "LOCAL_TASKS - '--repository' requires a non-empty option argument." + exit 1 + ;; + --snapshot) + # snapshot options, with value separated by space + if [ -n "$2" ]; then + option_snapshot="${2}" + shift + else + log_error "LOCAL_TASKS - '--snapshot' requires a non-empty option argument." + exit 1 + fi + ;; + --snapshot=?*) + # snapshot options, with value separated by = + option_snapshot="${1#*=}" + ;; + --snapshot=) + # snapshot options, without value + log_error "LOCAL_TASKS - '--snapshot' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - unkwnown option (ignored): '${1}'" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done ## Take a snapshot as a backup. ## Warning: You need to have a path.repo configured. ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes - curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" >> "${LOGFILE}" - curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" >> "${LOGFILE}" + local base_url="${option_protocol}://${option_host}:${option_port}" + local snapshot_url="${base_url}/_snapshot/${option_repository}/${option_snapshot}" + + if [ -n "${option_user}" ] || [ -n "${option_password}" ]; then + local option_auth="--user ${option_user}:${option_password}" + else + local option_auth="" + fi + + curl -s -XDELETE "${option_auth}" "${snapshot_url}" >> "${LOGFILE}" + curl -s -XPUT "${option_auth}" "${snapshot_url}?wait_for_completion=true" >> "${LOGFILE}" # Clustered version here # It basically the same thing except that you need to check that NFS is mounted # if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2' # then - # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" >> "${LOGFILE}" - # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" >> "${LOGFILE}" + # curl -s -XDELETE "${option_auth}" "${snapshot_url}" >> "${LOGFILE}" + # curl -s -XPUT "${option_auth}" "${snapshot_url}?wait_for_completion=true" >> "${LOGFILE}" # else # echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.' # fi - ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by : - # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do - # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}' - # done - # date=$(/bin/date +%F) - # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" >> "${LOGFILE}" + log "LOCAL_TASKS - stop dump_elasticsearch_snapshot_singlenode" +} - log "LOCAL_TASKS - stop dump_elasticsearch_snapshot" +####################################################################### +# Snapshot Elasticsearch data (multi-node cluster) +# +# Arguments: +# --protocol=[String] (default: http) +# --host=[String] (default: localhost) +# --port=[Integer] (default: 9200) +# --user=[String] (default: ) +# --password=[String] (default: ) +# --repository=[String] (default: snaprepo) +# --snapshot=[String] (default: snapshot.daily) +# --nfs-server=[IP|HOST] (default: ) +####################################################################### +dump_elasticsearch_snapshot_multinode() { + log "LOCAL_TASKS - start dump_elasticsearch_snapshot_multinode" + + local option_protocol="http" + local option_host="localhost" + local option_port="9200" + local option_user="" + local option_password="" + local option_repository="snaprepo" + local option_snapshot="snapshot.daily" + local option_nfs_server="" + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case $1 in + --protocol) + # protocol options, with value separated by space + if [ -n "$2" ]; then + option_protocol="${2}" + shift + else + log_error "LOCAL_TASKS - '--protocol' requires a non-empty option argument." + exit 1 + fi + ;; + --protocol=?*) + # protocol options, with value separated by = + option_protocol="${1#*=}" + ;; + --protocol=) + # protocol options, without value + log_error "LOCAL_TASKS - '--protocol' requires a non-empty option argument." + exit 1 + ;; + --host) + # host options, with value separated by space + if [ -n "$2" ]; then + option_host="${2}" + shift + else + log_error "LOCAL_TASKS - '--host' requires a non-empty option argument." + exit 1 + fi + ;; + --host=?*) + # host options, with value separated by = + option_host="${1#*=}" + ;; + --host=) + # host options, without value + log_error "LOCAL_TASKS - '--host' requires a non-empty option argument." + exit 1 + ;; + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - '--port' requires a non-empty option argument." + exit 1 + ;; + --user) + # user options, with value separated by space + if [ -n "$2" ]; then + option_user="${2}" + shift + else + log_error "LOCAL_TASKS - '--user' requires a non-empty option argument." + exit 1 + fi + ;; + --user=?*) + # user options, with value separated by = + option_user="${1#*=}" + ;; + --user=) + # user options, without value + log_error "LOCAL_TASKS - '--user' requires a non-empty option argument." + exit 1 + ;; + --password) + # password options, with value separated by space + if [ -n "$2" ]; then + option_password="${2}" + shift + else + log_error "LOCAL_TASKS - '--password' requires a non-empty option argument." + exit 1 + fi + ;; + --password=?*) + # password options, with value separated by = + option_password="${1#*=}" + ;; + --password=) + # password options, without value + log_error "LOCAL_TASKS - '--password' requires a non-empty option argument." + exit 1 + ;; + --repository) + # repository options, with value separated by space + if [ -n "$2" ]; then + option_repository="${2}" + shift + else + log_error "LOCAL_TASKS - '--repository' requires a non-empty option argument." + exit 1 + fi + ;; + --repository=?*) + # repository options, with value separated by = + option_repository="${1#*=}" + ;; + --repository=) + # repository options, without value + log_error "LOCAL_TASKS - '--repository' requires a non-empty option argument." + exit 1 + ;; + --snapshot) + # snapshot options, with value separated by space + if [ -n "$2" ]; then + option_snapshot="${2}" + shift + else + log_error "LOCAL_TASKS - '--snapshot' requires a non-empty option argument." + exit 1 + fi + ;; + --snapshot=?*) + # snapshot options, with value separated by = + option_snapshot="${1#*=}" + ;; + --snapshot=) + # snapshot options, without value + log_error "LOCAL_TASKS - '--snapshot' requires a non-empty option argument." + exit 1 + ;; + --nfs-server) + # nfs-server options, with value separated by space + if [ -n "$2" ]; then + option_nfs_server="${2}" + shift + else + log_error "LOCAL_TASKS - '--nfs-server' requires a non-empty option argument." + exit 1 + fi + ;; + --nfs-server=?*) + # nfs-server options, with value separated by = + option_nfs_server="${1#*=}" + ;; + --nfs-server=) + # nfs-server options, without value + log_error "LOCAL_TASKS - '--nfs-server' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - unkwnown option (ignored): '${1}'" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + ## Take a snapshot as a backup. + ## Warning: You need to have a path.repo configured. + ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes + + local base_url="${option_protocol}://${option_host}:${option_port}" + local snapshot_url="${base_url}/_snapshot/${option_repository}/${option_snapshot}" + + if [ -n "${option_user}" ] || [ -n "${option_password}" ]; then + local option_auth="--user ${option_user}:${option_password}" + else + local option_auth="" + fi + + # Clustered version here + # It basically the same thing except that you need to check that NFS is mounted + if ss | grep ':nfs' | grep -q -F "${option_nfs_server}"; then + curl -s -XDELETE "${option_auth}" "${snapshot_url}" >> "${LOGFILE}" + curl -s -XPUT "${option_auth}" "${snapshot_url}?wait_for_completion=true" >> "${LOGFILE}" + else + echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.' + fi + + log "LOCAL_TASKS - stop dump_elasticsearch_snapshot_multinode" } \ No newline at end of file diff --git a/client/zzz_evobackup.sh b/client/zzz_evobackup.sh index 3653d18..6b2dcba 100644 --- a/client/zzz_evobackup.sh +++ b/client/zzz_evobackup.sh @@ -137,22 +137,19 @@ local_tasks() { ########## MySQL ################## # Dump all grants (permissions), config variables and schema of databases - ### dump_mysql_meta + ### dump_mysql_meta [--port=3306] # Dump all databases in a single compressed file - ### dump_mysql_global + ### dump_mysql_global [--port=3306] [--masterdata] # Dump each database separately, in a compressed file - ### dump_mysql_per_base + ### dump_mysql_per_base [--port=3306] # Dump multiples instances, each in a single compressed file - ### dump_mysql_instances + ### dump_mysql_instance [--port=3306] # Dump each table in schema/data files, for all databases - ### dump_mysql_tabs - - # Run mysqlhotcopy for a specific database (must be configured) - # dump_mysql_hotcopy + ### dump_mysql_tabs [--port=3306] [--user=foo] [--password=123456789] ########## PostgreSQL ############# @@ -167,17 +164,20 @@ local_tasks() { ########## MongoDB ################ - ### dump_mongodb + ### dump_mongodb [--user=foo] [--password=123456789] ########## Redis ################## # Copy data file for all instances - ### dump_redis + ### dump_redis [--instances=] - ########## ElasticSearch ########## + ########## Elasticsearch ########## - # Trigger snapshots (must be configured) - ### dump_elasticsearch_snapshot + # Snapshot data for a single-node cluster + ### dump_elasticsearch_snapshot_singlenode [--protocol=http] [--host=localhost] [--port=9200] [--user=foo] [--password=123456789] [--repository=snaprepo] [--snapshot=snapshot.daily] + + # Snapshot data for a multi-node cluster + ### dump_elasticsearch_snapshot_multinode [--protocol=http] [--host=localhost] [--port=9200] [--user=foo] [--password=123456789] [--repository=snaprepo] [--snapshot=snapshot.daily] [--nfs-server=192.168.2.1] ########## RabbitMQ ############### @@ -191,7 +191,7 @@ local_tasks() { ########## Network ################ # Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) - dump_traceroute + dump_traceroute --targets=8.8.8.8,www.evolix.fr,travaux.evolix.net ########## Server state ########### From 1fa1eb77935b447362b85b7ed7f9db22f1264649 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 27 Feb 2023 14:56:45 +0100 Subject: [PATCH 13/20] Delete README containing dead links --- client/README.md | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 client/README.md diff --git a/client/README.md b/client/README.md deleted file mode 100644 index d1142f6..0000000 --- a/client/README.md +++ /dev/null @@ -1,3 +0,0 @@ -Pour l'installation de `zzz_evobackup`, voir - -Pour `update-evobackup-canary`, voir \ No newline at end of file From feafe016924d1f12be043e1c6eb75a6afadc2949 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 8 Mar 2023 09:22:28 +0100 Subject: [PATCH 14/20] Delete error directories recursively --- client/lib/main.sh | 4 ++-- client/zzz_evobackup | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/lib/main.sh b/client/lib/main.sh index b8723d0..52611a4 100644 --- a/client/lib/main.sh +++ b/client/lib/main.sh @@ -20,8 +20,8 @@ source "${LIBDIR}/dump.sh" local_tasks_wrapper() { log "START LOCAL_TASKS" - # Remove old log directories - find "${LOCAL_BACKUP_DIR}/" -type d -name "${PROGNAME}.errors-*" -ctime +30 -delete + # Remove old log directories (recursively) + find "${LOCAL_BACKUP_DIR}/" -type d -name "${PROGNAME}.errors-*" -ctime +30 -exec rm -rf \; local_tasks_type="$(type -t local_tasks)" if [ "${local_tasks_type}" = "function" ]; then diff --git a/client/zzz_evobackup b/client/zzz_evobackup index 0419cae..41c5ff6 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -114,7 +114,7 @@ lxc/*/rootfs/var/tmp local_tasks() { log "START LOCAL_TASKS" - # Remove old log directories + # Remove old log directories (recursively) find "${LOCAL_BACKUP_DIR}/" -type d -name "${PROGNAME}.errors-*" -ctime +30 -delete ################################################################### From 4475ee9af8a0aaab1d3f6f5353ce090149ad0b26 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 22 Mar 2023 14:10:11 +0100 Subject: [PATCH 15/20] dump.sh: improve options handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * default values, * reset variable each time * option masterdata seulement si présente --- client/lib/dump.sh | 76 +++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/client/lib/dump.sh b/client/lib/dump.sh index 9f1f3c2..6db0f9a 100644 --- a/client/lib/dump.sh +++ b/client/lib/dump.sh @@ -53,9 +53,9 @@ dump_mysql_global() { local option_port="3306" # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a while :; do - case $1 in + case ${1:-''} in --masterdata) - option_masterdata=1 + option_masterdata="--masterdata" ;; --port) # port options, with value separated by space @@ -95,14 +95,17 @@ dump_mysql_global() { done declare -a options - options+=("${option_masterdata}") + options=() options+=(--defaults-extra-file=/etc/mysql/debian.cnf) options+=(--port="${option_port}") options+=(--opt) - options+=(--all-databases) options+=(--force) options+=(--events) options+=(--hex-blob) + options+=(--all-databases) + if [ -n "${option_masterdata}" ]; then + options+=("${option_masterdata}") + fi mysqldump "${options[@]}" 2> "${error_file}" | gzip --best > "${dump_file}" @@ -133,7 +136,7 @@ dump_mysql_per_base() { local option_port="3306" # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a while :; do - case $1 in + case ${1:-''} in --port) # port options, with value separated by space if [ -n "$2" ]; then @@ -172,6 +175,7 @@ dump_mysql_per_base() { done declare -a options + options=() options+=(--defaults-extra-file=/etc/mysql/debian.cnf) options+=(--port="${option_port}") options+=(--force) @@ -214,7 +218,7 @@ dump_mysql_meta() { local option_port="3306" # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a while :; do - case $1 in + case ${1:-''} in --port) # port options, with value separated by space if [ -n "$2" ]; then @@ -258,6 +262,7 @@ dump_mysql_meta() { log "LOCAL_TASKS - start ${dump_file}" declare -a options + options=() options+=(--port "${option_port}") options+=(--flush) options+=(--no-header) @@ -280,8 +285,9 @@ dump_mysql_meta() { log "LOCAL_TASKS - start ${dump_file}" declare -a options + options=() options+=(--port="${option_port}") - options+=(-A) + options+=(--no-auto-rehash) options+=(-e "SHOW GLOBAL VARIABLES;") mysql "${options[@]}" 2> "${error_file}" > "${dump_file}" @@ -304,6 +310,7 @@ dump_mysql_meta() { log "LOCAL_TASKS - start ${dump_file}" declare -a options + options=() options+=(--defaults-extra-file=/etc/mysql/debian.cnf) options+=(--port="${option_port}") options+=(--force) @@ -346,7 +353,7 @@ dump_mysql_tabs() { local option_port="3306" # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a while :; do - case $1 in + case ${1:-''} in --port) # port options, with value separated by space if [ -n "$2" ]; then @@ -386,6 +393,7 @@ dump_mysql_tabs() { done declare -a options + options=() options+=(--defaults-extra-file=/etc/mysql/debian.cnf) options+=(--port="${option_port}") options+=(--force) @@ -434,7 +442,7 @@ dump_mysql_instance() { local option_password="" # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a while :; do - case $1 in + case ${1:-''} in --port) # port options, with value separated by space if [ -n "$2" ]; then @@ -511,6 +519,7 @@ dump_mysql_instance() { done declare -a options + options=() options+=(--port="${option_port}") options+=(--user="${option_user}") options+=(--password="${option_password}") @@ -660,7 +669,7 @@ dump_redis() { local option_instances="" # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a while :; do - case $1 in + case ${1:-''} in --instances) # instances options, with key and value separated by space if [ -n "$2" ]; then @@ -761,7 +770,7 @@ dump_mongodb() { local option_password="" # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a while :; do - case $1 in + case ${1:-''} in --user) # user options, with value separated by space if [ -n "$2" ]; then @@ -819,6 +828,7 @@ dump_mongodb() { done declare -a options + options=() options+=(--username="${option_user}") options+=(--password="${option_password}") options+=(--out="${dump_dir}/") @@ -881,7 +891,7 @@ dump_traceroute() { local option_targets="" # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a while :; do - case $1 in + case ${1:-''} in --targets) # targets options, with key and value separated by space if [ -n "$2" ]; then @@ -947,10 +957,9 @@ dump_traceroute() { ####################################################################### # Save many system information, using dump_server_state # -# Arguments: -# -# TODO: pass arguments to the dump_server_state command -# with defaults and overrides (incuding "dump-dir") +# Arguments: +# any option for dump-server-state (except --dump-dir) is usable +# (default: --all) ####################################################################### dump_server_state() { local dump_dir="${LOCAL_BACKUP_DIR}/server-state" @@ -961,27 +970,26 @@ dump_server_state() { log "LOCAL_TASKS - start ${dump_dir}" + # pass all options + read -a options <<< "${@}" + # if no option is given, use "--all" as fallback + if [ ${#options[@]} -le 0 ]; then + options=(--all) + fi + # add "--dump-dir" in case it is missing (as it should) + options+=(--dump-dir "${dump_dir}") + dump_server_state_bin=$(command -v dump-server-state) if [ -z "${dump_server_state_bin}" ]; then log_error "LOCAL_TASKS - dump-server-state is missing" rc=1 else - if [ "${SYSTEM}" = "linux" ]; then - ${dump_server_state_bin} --all --dump-dir "${dump_dir}" - local last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - dump-server-state returned an error ${last_rc}, check ${dump_dir}" - GLOBAL_RC=${E_DUMPFAILED} - fi - else - ${dump_server_state_bin} --all --dump-dir "${dump_dir}" - local last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - dump-server-state returned an error ${last_rc}, check ${dump_dir}" - GLOBAL_RC=${E_DUMPFAILED} - fi + ${dump_server_state_bin} "${options[@]}" + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - dump-server-state returned an error ${last_rc}, check ${dump_dir}" + GLOBAL_RC=${E_DUMPFAILED} fi fi log "LOCAL_TASKS - stop ${dump_dir}" @@ -1064,7 +1072,7 @@ dump_elasticsearch_snapshot_singlenode() { local option_snapshot="snapshot.daily" # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a while :; do - case $1 in + case ${1:-''} in --protocol) # protocol options, with value separated by space if [ -n "$2" ]; then @@ -1271,7 +1279,7 @@ dump_elasticsearch_snapshot_multinode() { local option_nfs_server="" # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a while :; do - case $1 in + case ${1:-''} in --protocol) # protocol options, with value separated by space if [ -n "$2" ]; then From 5aeba28d5c6cd4982a08a1c72190f2b3ab63fdae Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 22 Mar 2023 14:10:27 +0100 Subject: [PATCH 16/20] utilities.sh: fix line count --- client/lib/utilities.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/lib/utilities.sh b/client/lib/utilities.sh index 0f48077..4b07c86 100644 --- a/client/lib/utilities.sh +++ b/client/lib/utilities.sh @@ -16,7 +16,7 @@ log_error() { if [ -n "${error_file}" ] && [ -f "${error_file}" ]; then printf "\n### %s\n" "${error_msg}" >&2 # shellcheck disable=SC2046 - if [ $(wc -l "${error_file}") -gt 30 ]; then + if [ $(wc -l "${error_file}" | cut -d " " -f 1) -gt 30 ]; then printf "~~~{%s (tail -30)}\n" "${error_file}" >&2 tail -n 30 "${error_file}" >&2 else From 70e541dd6d23c430fe5bd775fffc2041a0a1df1f Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 22 Mar 2023 14:11:03 +0100 Subject: [PATCH 17/20] zzz_evobackup.sh: LIBDIR="/usr/local/lib/evobackup" --- client/zzz_evobackup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/zzz_evobackup.sh b/client/zzz_evobackup.sh index 6b2dcba..f6bf335 100644 --- a/client/zzz_evobackup.sh +++ b/client/zzz_evobackup.sh @@ -261,7 +261,7 @@ setup_custom() { ########## Libraries ################################################## # Change this to wherever you install the libraries -LIBDIR="./lib" +LIBDIR="/usr/local/lib/evobackup" source "${LIBDIR}/main.sh" From ea054f314c42cdaefba13661dd841b5fac232765 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 22 Mar 2023 14:17:10 +0100 Subject: [PATCH 18/20] Add some comments --- client/zzz_evobackup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/zzz_evobackup.sh b/client/zzz_evobackup.sh index f6bf335..9afe3d2 100644 --- a/client/zzz_evobackup.sh +++ b/client/zzz_evobackup.sh @@ -191,11 +191,13 @@ local_tasks() { ########## Network ################ # Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) + ### dump_traceroute --targets=host_or_ip[,host_or_ip] dump_traceroute --targets=8.8.8.8,www.evolix.fr,travaux.evolix.net ########## Server state ########### # Run dump-server-state to extract system information + ### dump-server-state [any dump-server-state option] dump_server_state # Dump file access control lists From a6573c6db324f0c699cf60e3cdc64b091ba9b7de Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 22 Mar 2023 14:17:42 +0100 Subject: [PATCH 19/20] changelog --- client/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/CHANGELOG.md b/client/CHANGELOG.md index 06b5faa..d898560 100644 --- a/client/CHANGELOG.md +++ b/client/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +* split functions into libraries + ### Deprecated ### Removed From 5ac990473ebc26c5ed6de796771019ac73aafb4d Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 22 Mar 2023 14:19:29 +0100 Subject: [PATCH 20/20] remove monolithic script --- client/zzz_evobackup | 1232 ++++++--------------------------------- client/zzz_evobackup.sh | 272 --------- 2 files changed, 172 insertions(+), 1332 deletions(-) mode change 100755 => 100644 client/zzz_evobackup delete mode 100644 client/zzz_evobackup.sh diff --git a/client/zzz_evobackup b/client/zzz_evobackup old mode 100755 new mode 100644 index 41c5ff6..9afe3d2 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -14,115 +14,121 @@ # # Licence: AGPLv3 -VERSION="22.12" - -##### CONFIGURATION ################################################### +####################################################################### # -# 1. Set the following MAIL and SERVERS variables. -# 2. Customize the RSYNC_INCLUDES and RSYNC_EXCLUDES variables. -# 3. Enable or disable local tasks inside the local_tasks() function. +# You must configure the MAIL variable to receive notifications. # -# Some local tasks are configurable. -# If you enable them, have a look at their implementation. +# There is some optional configuration that you can do +# at the end of this script. # -# Some additional configuration variable can be customized -# at the end of the script, before invoking the main() function. +# The library (usually installed at /usr/local/lib/evobackup/main.sh) +# also has many variables that you can override for fine-tuning. # ####################################################################### -# email adress for notifications +# Email adress for notifications MAIL=jdoe@example.com -# list of hosts (hostname or IP) and SSH port for Rsync -SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX" +####################################################################### +# +# The "sync_tasks" function will be called by the main function. +# +# You can customize the variables: +# * "sync_name" (String) +# * "SERVERS" (Array of HOST:PORT) +# * "RSYNC_INCLUDES" (Array of paths to include) +# * "RSYNC_EXCLUDES" (Array of paths to exclude) +# +# The "sync" function can be called multiple times +# with a different set of variables. +# That way you can to sync to various destinations. +# +####################################################################### -# We use /home/backup : feel free to use your own dir -LOCAL_BACKUP_DIR="/home/backup" +sync_tasks() { -# Enable/disable local tasks (default: enabled) -: "${LOCAL_TASKS:=1}" -# Enable/disable sync tasks (default: enabled) -: "${SYNC_TASKS:=1}" -# Enable/disable mtree (default: enabled) -: "${MTREE_ENABLED:=1}" + ########## System-only backup (to Evolix servers) ################# -# Source paths can be customized -# Empty lines, and lines containing # or ; are ignored -RSYNC_INCLUDES=" -/etc -/root -/var -/home -" + # Name your sync task, for logs + sync_name="evolix-system" -# Excluded paths can be customized -# Empty lines, and lines beginning with # or ; are ignored -RSYNC_EXCLUDES=" -/dev -/proc -/run -/sys -/tmp -/usr/doc -/usr/obj -/usr/share/doc -/usr/src -/var/apt -/var/cache -/var/db/munin/*.tmp -/var/lib/amavis/amavisd.sock -/var/lib/amavis/tmp -/var/lib/clamav/*.tmp -/var/lib/elasticsearch -/var/lib/metche -/var/lib/mongodb -/var/lib/munin/*tmp* -/var/lib/mysql -/var/lib/php/sessions -/var/lib/php5 -/var/lib/postgres -/var/lib/postgresql -/var/lib/sympa -/var/lock -/var/run -/var/spool/postfix -/var/spool/smtpd -/var/spool/squid -/var/state -/var/tmp -lost+found -.nfs.* -lxc/*/rootfs/tmp -lxc/*/rootfs/usr/doc -lxc/*/rootfs/usr/obj -lxc/*/rootfs/usr/share/doc -lxc/*/rootfs/usr/src -lxc/*/rootfs/var/apt -lxc/*/rootfs/var/cache -lxc/*/rootfs/var/lib/php5 -lxc/*/rootfs/var/lib/php/sessions -lxc/*/rootfs/var/lock -lxc/*/rootfs/var/run -lxc/*/rootfs/var/state -lxc/*/rootfs/var/tmp -/home/mysqltmp -" + # List of host/port for your sync task + # shellcheck disable=SC2034 + SERVERS=( + node0.backup.evolix.net:2234 + node1.backup.evolix.net:2234 + ) -##### FUNCTIONS ####################################################### + # What to include in your sync task + # Add or remove paths if you need + # shellcheck disable=SC2034 + RSYNC_INCLUDES=( + "${rsync_default_includes[@]}" + /etc + /root + /var + ) + + # What to exclude from your sync task + # Add or remove paths if you need + # shellcheck disable=SC2034 + RSYNC_EXCLUDES=( + "${rsync_default_excludes[@]}" + ) + + # Call the sync task + sync "${sync_name}" "SERVERS[@]" "RSYNC_INCLUDES[@]" "RSYNC_EXCLUDES[@]" + + + ########## Full backup (to client servers) ######################## + + # Name your sync task, for logs + sync_name="client-full" + + # List of host/port for your sync task + # shellcheck disable=SC2034 + SERVERS=( + client-backup00.evolix.net:2221 + client-backup01.evolix.net:2221 + ) + + # What to include in your sync task + # Add or remove paths if you need + # shellcheck disable=SC2034 + RSYNC_INCLUDES=( + "${rsync_default_includes[@]}" + /etc + /root + /var + /home + /srv + ) + + # What to exclude from your sync task + # Add or remove paths if you need + # shellcheck disable=SC2034 + RSYNC_EXCLUDES=( + "${rsync_default_excludes[@]}" + ) + + # Call the sync task + sync "${sync_name}" "SERVERS[@]" "RSYNC_INCLUDES[@]" "RSYNC_EXCLUDES[@]" + +} + +####################################################################### +# +# The "local_tasks" function will be called by the main function. +# +# You can call any available "dump_xxx" function +# (usually installed at /usr/local/lib/evobackup/dump.sh) +# +# You can also write some custom functions and call them. +# A "dump_custom" example is available further down. +# +####################################################################### -# Execute all local tasks: database dumps, system state dump… local_tasks() { - log "START LOCAL_TASKS" - - # Remove old log directories (recursively) - find "${LOCAL_BACKUP_DIR}/" -type d -name "${PROGNAME}.errors-*" -ctime +30 -delete - - ################################################################### - # You can enable/disable local tasks - # by (un)commenting calls to "dump_XXX" functions. - # - # You can also add your own functions and call them from here. - ################################################################### ########## OpenLDAP ############### @@ -131,22 +137,19 @@ local_tasks() { ########## MySQL ################## # Dump all grants (permissions), config variables and schema of databases - ### dump_mysql_meta + ### dump_mysql_meta [--port=3306] # Dump all databases in a single compressed file - ### dump_mysql_global + ### dump_mysql_global [--port=3306] [--masterdata] # Dump each database separately, in a compressed file - ### dump_mysql_per_base + ### dump_mysql_per_base [--port=3306] # Dump multiples instances, each in a single compressed file - ### dump_mysql_instances + ### dump_mysql_instance [--port=3306] # Dump each table in schema/data files, for all databases - ### dump_mysql_tabs - - # Run mysqlhotcopy for a specific database (must be configured) - # dump_mysql_hotcopy + ### dump_mysql_tabs [--port=3306] [--user=foo] [--password=123456789] ########## PostgreSQL ############# @@ -161,17 +164,20 @@ local_tasks() { ########## MongoDB ################ - ### dump_mongodb + ### dump_mongodb [--user=foo] [--password=123456789] ########## Redis ################## # Copy data file for all instances - ### dump_redis + ### dump_redis [--instances=] - ########## ElasticSearch ########## + ########## Elasticsearch ########## - # Trigger snapshots (must be configured) - ### dump_elasticsearch_snapshot + # Snapshot data for a single-node cluster + ### dump_elasticsearch_snapshot_singlenode [--protocol=http] [--host=localhost] [--port=9200] [--user=foo] [--password=123456789] [--repository=snaprepo] [--snapshot=snapshot.daily] + + # Snapshot data for a multi-node cluster + ### dump_elasticsearch_snapshot_multinode [--protocol=http] [--host=localhost] [--port=9200] [--user=foo] [--password=123456789] [--repository=snaprepo] [--snapshot=snapshot.daily] [--nfs-server=192.168.2.1] ########## RabbitMQ ############### @@ -185,976 +191,82 @@ local_tasks() { ########## Network ################ # Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) - dump_traceroute + ### dump_traceroute --targets=host_or_ip[,host_or_ip] + dump_traceroute --targets=8.8.8.8,www.evolix.fr,travaux.evolix.net ########## Server state ########### # Run dump-server-state to extract system information + ### dump-server-state [any dump-server-state option] dump_server_state # Dump file access control lists ### dump_facl - ################################################################### - - print_error_files_content - - log "STOP LOCAL_TASKS" + # No-op, in case nothing is enabled + : } -# Output error files content, if any -print_error_files_content() { - # Search for error files - error_files=$(find "${ERRORS_DIR}" -type f) - for error_file in ${error_files}; do - error_file_size=$(stat -c "%s" "${error_file}") +# This is an example for a custom dump function +# Uncomment, customize and call it from the "local_tasks" function +### dump_custom() { +### # Set dump and errors directories and files +### local dump_dir="${LOCAL_BACKUP_DIR}/custom" +### local dump_file="${dump_dir}/dump.gz" +### local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") +### local error_file="${errors_dir}/dump.err" +### +### # Reset dump and errors directories +### rm -rf "${dump_dir}" "${errors_dir}" +### # shellcheck disable=SC2174 +### mkdir -p -m 700 "${dump_dir}" "${errors_dir}" +### +### # Log the start of the command +### log "LOCAL_TASKS - start ${dump_file}" +### +### # Execute your dump command +### # Send errors to the error file and the data to the dump file +### my-dump-command 2> "${error_file}" > "${dump_file}" +### +### # Check result and deal with potential errors +### local last_rc=$? +### # shellcheck disable=SC2086 +### if [ ${last_rc} -ne 0 ]; then +### log_error "LOCAL_TASKS - my-dump-command to ${dump_file} returned an error ${last_rc}" "${error_file}" +### GLOBAL_RC=${E_DUMPFAILED} +### else +### rm -f "${error_file}" +### fi +### +### # Log the end of the command +### log "LOCAL_TASKS - stop ${dump_file}" +### } - # shellcheck disable=SC2086 - if [ ${error_file_size} -gt 0 ]; then - # if the file is not empty, display its content - printf "### cat %s ###\n" "${error_file}" >&2 - cat "${error_file}" >&2 - else - # if the file is empty, remove it - rm "${error_file}" - fi - done +########## Optional configuration ##################################### - # Search for remaining error_files - error_files_count=$(find "${ERRORS_DIR}" -type f | wc -l) - # If there is no error file, clean the parent directories - if [ "${error_files_count}" -eq 0 ]; then - rm -rf "${ERRORS_DIR}" - fi -} -# shellcheck disable=SC2317 -mysql_list_databases() { - port=${1:-"3306"} +setup_custom() { + # If you set a value (like "linux", "openbsd"…) it will be used, + # Default: uname(1) in lowercase. + ### SYSTEM="linux" - mysql --defaults-extra-file=/etc/mysql/debian.cnf --port="${port}" --execute="show databases" --silent --skip-column-names \ - | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)" + # If you set a value it will be used, + # Default: hostname(1). + ### HOSTNAME="example-host" + + # Email subect for notifications + ### MAIL_SUBJECT="[info] EvoBackup - Client ${HOSTNAME}" + + # No-op in case nothing is executed + : } -# shellcheck disable=SC2317 -errors_dir_from_dump_dir() { - realpath --canonicalize-missing "${ERRORS_DIR}/$(realpath --relative-to="${LOCAL_BACKUP_DIR}" "${1}")" -} -# shellcheck disable=SC2317 -dump_ldap() { - ## OpenLDAP : example with slapcat - dump_dir="${LOCAL_BACKUP_DIR}/ldap" - rm -rf "${dump_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" +########## Libraries ################################################## - log "LOCAL_TASKS - start dump_ldap to ${dump_dir}" +# Change this to wherever you install the libraries +LIBDIR="/usr/local/lib/evobackup" - slapcat -n 0 -l "${dump_dir}/config.bak" - slapcat -n 1 -l "${dump_dir}/data.bak" - slapcat -l "${dump_dir}/all.bak" +source "${LIBDIR}/main.sh" - log "LOCAL_TASKS - stop dump_ldap" -} -# shellcheck disable=SC2317 -dump_mysql_global() { - dump_dir="${LOCAL_BACKUP_DIR}/mysql-global" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" +########## Let's go! ################################################## - error_file="${errors_dir}/mysql.bak.err" - dump_file="${dump_dir}/mysql.bak.gz" - log "LOCAL_TASKS - start ${dump_file}" - - mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 --opt --all-databases --force --events --hex-blob 2> "${error_file}" | gzip --best > "${dump_file}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_file}" -} -# shellcheck disable=SC2317 -dump_mysql_per_base() { - dump_dir="${LOCAL_BACKUP_DIR}/mysql-per-base" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - databases=$(mysql_list_databases 3306) - for database in ${databases}; do - error_file="${errors_dir}/${database}.err" - dump_file="${dump_dir}/${database}.sql.gz" - log "LOCAL_TASKS - start ${dump_file}" - - mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob "${database}" 2> "${error_file}" | gzip --best > "${dump_file}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_file}" - done -} -# shellcheck disable=SC2317 -dump_mysql_meta() { - dump_dir="${LOCAL_BACKUP_DIR}/mysql-meta" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - ## Dump all grants (requires 'percona-toolkit' package) - error_file="${errors_dir}/all_grants.err" - dump_file="${dump_dir}/all_grants.sql" - log "LOCAL_TASKS - start ${dump_file}" - - pt-show-grants --flush --no-header 2> "${error_file}" > "${dump_file}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - pt-show-grants to ${dump_file} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_file}" - - ## Dump all variables - error_file="${errors_dir}/variables.err" - dump_file="${dump_dir}/variables.txt" - log "LOCAL_TASKS - start ${dump_file}" - - mysql -A -e "SHOW GLOBAL VARIABLES;" 2> "${error_file}" > "${dump_file}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - mysql 'show variables' returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_file}" - - ## Schema only (no data) for each databases - databases=$(mysql_list_databases 3306) - for database in ${databases}; do - error_file="${errors_dir}/${database}.schema.err" - dump_file="${dump_dir}/${database}.schema.sql" - log "LOCAL_TASKS - start ${dump_file}" - - mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases "${database}" 2> "${error_file}" > "${dump_file}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_file}" - done -} -# shellcheck disable=SC2317 -dump_mysql_tabs() { - databases=$(mysql_list_databases 3306) - for database in ${databases}; do - dump_dir="${LOCAL_BACKUP_DIR}/mysql-tabs/${database}" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - chown -RL mysql "${dump_dir}" - - error_file="${errors_dir}.err" - log "LOCAL_TASKS - start ${dump_dir}" - - mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments --fields-enclosed-by='\"' --fields-terminated-by=',' -T "${dump_dir}" "${database}" 2> "${error_file}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - mysqldump to ${dump_dir} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_dir}" - done -} -# shellcheck disable=SC2317 -dump_mysql_hotcopy() { - # customize the list of databases to hot-copy - databases="" - for database in ${databases}; do - dump_dir="${LOCAL_BACKUP_DIR}/mysql-hotcopy/${database}" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - error_file="${errors_dir}.err" - log "LOCAL_TASKS - start ${dump_dir}" - - mysqlhotcopy "${database}" "${dump_dir}/" 2> "${error_file}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - mysqlhotcopy to ${dump_dir} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_dir}" - done -} -# shellcheck disable=SC2317 -dump_mysql_instances() { - dump_dir="${LOCAL_BACKUP_DIR}/mysql-instances" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - mysql_user="mysqladmin" - mysql_passwd=$(grep -m1 'password = .*' /root/.my.cnf | cut -d " " -f 3) - - # customize list of instances - instances="" - for instance in ${instances}; do - error_file="${errors_dir}/${instance}.err" - dump_file="${dump_dir}/${instance}.bak.gz" - log "LOCAL_TASKS - start ${dump_file}" - - mysqldump --port="${instance}" --opt --all-databases --hex-blob --user="${mysql_user}" --password="${mysql_passwd}" 2> "${error_file}" | gzip --best > "${dump_file}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_file}" - done -} -# shellcheck disable=SC2317 -dump_postgresql_global() { - dump_dir="${LOCAL_BACKUP_DIR}/postgresql-global" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - ## example with pg_dumpall and with compression - dump_file="${dump_dir}/pg.dump.bak.gz" - log "LOCAL_TASKS - start ${dump_file}" - - (sudo -u postgres pg_dumpall) 2> "${error_file}" | gzip --best > "${dump_file}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - pg_dumpall to ${dump_file} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - - log "LOCAL_TASKS - stop ${dump_file}" - - ## example with pg_dumpall and without compression - ## WARNING: you need space in ~postgres - # dump_file="${dump_dir}/pg.dump.bak" - # log "LOCAL_TASKS - start ${dump_file}" - # - # (su - postgres -c "pg_dumpall > ~/pg.dump.bak") 2> "${error_file}" - # mv ~postgres/pg.dump.bak "${dump_file}" - # - # log "LOCAL_TASKS - stop ${dump_file}" -} -# shellcheck disable=SC2317 -dump_postgresql_per_base() { - dump_dir="${LOCAL_BACKUP_DIR}/postgresql-per-base" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - ( - # shellcheck disable=SC2164 - cd /var/lib/postgresql - databases=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' | grep -v "template.*") - for database in ${databases} ; do - error_file="${errors_dir}/${database}.err" - dump_file="${dump_dir}/${database}.sql.gz" - log "LOCAL_TASKS - start ${dump_file}" - - (sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d "${database}") 2> "${error_file}" | gzip --best > "${dump_file}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - pg_dump to ${dump_file} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_file}" - done - ) -} -# shellcheck disable=SC2317 -dump_postgresql_filtered() { - dump_dir="${LOCAL_BACKUP_DIR}/postgresql-filtered" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - error_file="${errors_dir}/pg-backup.err" - dump_file="${dump_dir}/pg-backup.tar" - log "LOCAL_TASKS - start ${dump_file}" - - ## example with all tables from MYBASE excepts TABLE1 and TABLE2 - # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f "${dump_file}" -t 'TABLE1' -t 'TABLE2' MYBASE 2> "${error_file}" - - ## example with only TABLE1 and TABLE2 from MYBASE - # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f "${dump_file}" -T 'TABLE1' -T 'TABLE2' MYBASE 2> "${error_file}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - pg_dump to ${dump_file} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_file}" -} -# shellcheck disable=SC2317 -dump_redis() { - instances=$(find /var/lib/ -mindepth 1 -maxdepth 1 -type d -name 'redis*') - for instance in ${instances}; do - name=$(basename "${instance}") - dump_dir="${LOCAL_BACKUP_DIR}/${name}" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - if [ -f "${instance}/dump.rdb" ]; then - error_file="${errors_dir}/${instance}.err" - log "LOCAL_TASKS - start ${dump_dir}" - - cp -a "${instance}/dump.rdb" "${dump_dir}/" 2> "${error_file}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - cp ${instance}/dump.rdb to ${dump_dir} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_dir}" - fi - done -} -# shellcheck disable=SC2317 -dump_mongodb() { - ## don't forget to create use with read-only access - ## > use admin - ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } ) - - dump_dir="${LOCAL_BACKUP_DIR}/mongodump" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - error_file="${errors_dir}.err" - log "LOCAL_TASKS - start ${dump_dir}" - - mongo_user="" - mongo_password="" - - mongodump -u "${mongo_user}" -p"${mongo_password}" -o "${dump_dir}/" 2> "${error_file}" > /dev/null - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - mongodump to ${dump_dir} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_dir}" -} -# shellcheck disable=SC2317 -dump_megacli_config() { - dump_dir="${LOCAL_BACKUP_DIR}/megacli" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - dump_file="${dump_dir}/megacli.cfg" - error_file="${errors_dir}/megacli.err" - log "LOCAL_TASKS - start ${dump_file}" - - megacli -CfgSave -f "${dump_file}" -a0 2> "${error_file}" > /dev/null - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - megacli to ${dump_file} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_file}" -} -# shellcheck disable=SC2317 -dump_traceroute() { - dump_dir="${LOCAL_BACKUP_DIR}/traceroute" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - network_targets="8.8.8.8 www.evolix.fr travaux.evolix.net" - - mtr_bin=$(command -v mtr) - if [ -n "${network_targets}" ] && [ -n "${mtr_bin}" ]; then - for addr in ${network_targets}; do - dump_file="${dump_dir}/mtr-${addr}" - log "LOCAL_TASKS - start ${dump_file}" - - ${mtr_bin} -r "${addr}" > "${dump_file}" - - log "LOCAL_TASKS - stop ${dump_file}" - done - fi - - traceroute_bin=$(command -v traceroute) - if [ -n "${network_targets}" ] && [ -n "${traceroute_bin}" ]; then - for addr in ${network_targets}; do - dump_file="${dump_dir}/traceroute-${addr}" - log "LOCAL_TASKS - start ${dump_file}" - - ${traceroute_bin} -n "${addr}" > "${dump_file}" 2>&1 - - log "LOCAL_TASKS - stop ${dump_file}" - done - fi -} -# shellcheck disable=SC2317 -dump_server_state() { - dump_dir="${LOCAL_BACKUP_DIR}/server-state" - rm -rf "${dump_dir}" - # Do not create the directory - # shellcheck disable=SC2174 - # mkdir -p -m 700 "${dump_dir}" - - log "LOCAL_TASKS - start ${dump_dir}" - - dump_server_state_bin=$(command -v dump-server-state) - if [ -z "${dump_server_state_bin}" ]; then - log_error "LOCAL_TASKS - dump-server-state is missing" - rc=1 - else - if [ "${SYSTEM}" = "linux" ]; then - ${dump_server_state_bin} --all --dump-dir "${dump_dir}" - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - dump-server-state returned an error ${last_rc}, check ${dump_dir}" - rc=${E_DUMPFAILED} - fi - else - ${dump_server_state_bin} --all --dump-dir "${dump_dir}" - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - dump-server-state returned an error ${last_rc}, check ${dump_dir}" - rc=${E_DUMPFAILED} - fi - fi - fi - log "LOCAL_TASKS - stop ${dump_dir}" -} -# shellcheck disable=SC2317 -dump_rabbitmq() { - dump_dir="${LOCAL_BACKUP_DIR}/rabbitmq" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - error_file="${errors_dir}.err" - dump_file="${dump_dir}/config" - log "LOCAL_TASKS - start ${dump_file}" - - rabbitmqadmin export "${dump_file}" 2> "${error_file}" >> "${LOGFILE}" - - last_rc=$? - # shellcheck disable=SC2086 - if [ ${last_rc} -ne 0 ]; then - log_error "LOCAL_TASKS - pg_dump to ${dump_file} returned an error ${last_rc}" "${error_file}" - rc=${E_DUMPFAILED} - else - rm -f "${error_file}" - fi - log "LOCAL_TASKS - stop ${dump_file}" -} -# shellcheck disable=SC2317 -dump_facl() { - dump_dir="${LOCAL_BACKUP_DIR}/facl" - errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") - rm -rf "${dump_dir}" "${errors_dir}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${dump_dir}" "${errors_dir}" - - log "LOCAL_TASKS - start ${dump_dir}" - - getfacl -R /etc > "${dump_dir}/etc.txt" - getfacl -R /home > "${dump_dir}/home.txt" - getfacl -R /usr > "${dump_dir}/usr.txt" - getfacl -R /var > "${dump_dir}/var.txt" - - log "LOCAL_TASKS - stop ${dump_dir}" -} -# shellcheck disable=SC2317 -dump_elasticsearch_snapshot() { - log "LOCAL_TASKS - start dump_elasticsearch_snapshot" - - ## Take a snapshot as a backup. - ## Warning: You need to have a path.repo configured. - ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes - - curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" >> "${LOGFILE}" - curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" >> "${LOGFILE}" - - # Clustered version here - # It basically the same thing except that you need to check that NFS is mounted - # if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2' - # then - # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" >> "${LOGFILE}" - # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" >> "${LOGFILE}" - # else - # echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.' - # fi - - ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by : - # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do - # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}' - # done - # date=$(/bin/date +%F) - # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" >> "${LOGFILE}" - - log "LOCAL_TASKS - stop dump_elasticsearch_snapshot" -} - -# Call test_server with "HOST:PORT" string -# It will return with 0 if the server is reachable. -# It will return with 1 and a message on stderr if not. -test_server() { - item=$1 - # split HOST and PORT from the input string - host=$(echo "${item}" | cut -d':' -f1) - port=$(echo "${item}" | cut -d':' -f2) - - # Test if the server is accepting connections - ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" "${host}" -p "${port}" -t "exit" - # shellcheck disable=SC2181 - if [ $? = 0 ]; then - # SSH connection is OK - return 0 - else - # SSH connection failed - new_error=$(printf "Failed to connect to \`%s' within %s seconds" "${item}" "${SSH_CONNECT_TIMEOUT}") - log "${new_error}" - SERVERS_SSH_ERRORS=$(printf "%s\\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d') - - return 1 - fi -} -# Call pick_server with an optional positive integer to get the nth server in the list. -pick_server() { - increment=${1:-0} - list_length=$(echo "${SERVERS}" | wc -w) - - if [ "${increment}" -ge "${list_length}" ]; then - # We've reached the end of the list - new_error="No more server available" - log "${new_error}" - SERVERS_SSH_ERRORS=$(printf "%s\\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d') - - # Log errors to stderr - printf "%s\\n" "${SERVERS_SSH_ERRORS}" >&2 - return 1 - fi - - # Extract the day of month, without leading 0 (which would give an octal based number) - today=$(/bin/date +%e) - # A salt is useful to randomize the starting point in the list - # but stay identical each time it's called for a server (based on hostname). - salt=$(hostname | cksum | cut -d' ' -f1) - # Pick an integer between 0 and the length of the SERVERS list - # It changes each day - item=$(( (today + salt + increment) % list_length )) - # cut starts counting fields at 1, not 0. - field=$(( item + 1 )) - - echo "${SERVERS}" | cut -d ' ' -f ${field} -} - -sync_tasks() { - n=0 - server="" - if [ "${SERVERS_FALLBACK}" = "1" ]; then - # We try to find a suitable server - while :; do - server=$(pick_server "${n}") - test $? = 0 || exit ${E_NOSRVAVAIL} - - if test_server "${server}"; then - break - else - server="" - n=$(( n + 1 )) - fi - done - else - # we force the server - server=$(pick_server "${n}") - fi - - SSH_SERVER=$(echo "${server}" | cut -d':' -f1) - SSH_PORT=$(echo "${server}" | cut -d':' -f2) - - log "START SYNC_TASKS - server=${server}" - - # default paths, depending on system - if [ "${SYSTEM}" = "linux" ]; then - rsync_default_includes=(/bin /boot /lib /opt /sbin /usr) - else - rsync_default_includes=(/bsd /bin /sbin /usr) - fi - if [ -f "${CANARY_FILE}" ]; then - rsync_default_includes+=("${CANARY_FILE}") - fi - - # reset Rsync log file - if [ -n "$(command -v truncate)" ]; then - truncate -s 0 "${RSYNC_LOGFILE}" - else - printf "" > "${RSYNC_LOGFILE}" - fi - - # Create a temp file for excludes and includes - rsync_includes_file="$(mktemp --tmpdir "${PROGNAME}.rsync-includes.XXXXXX")" - rsync_excludes_file="$(mktemp --tmpdir "${PROGNAME}.rsync-excludes.XXXXXX")" - # … and add them to the list of files to delete at exit - temp_files+=("${rsync_includes_file}") - temp_files+=("${rsync_excludes_file}") - - # Store includes/excludes in files - # without blank lines of comments (# or ;) - echo "${RSYNC_INCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${rsync_includes_file}" - echo "${RSYNC_EXCLUDES}" | sed -e 's/\s*\(#\|;\).*//; /^\s*$/d' > "${rsync_excludes_file}" - - if [ "${MTREE_ENABLED}" = "1" ]; then - mtree_bin=$(command -v mtree) - - if [ -n "${mtree_bin}" ]; then - # Dump filesystem stats with mtree - log "SYNC_TASKS - start mtree" - - mtree_files=() - - # Loop over Rsync includes - while read -r line ; do - # … but exclude for mtree what will be excluded by Rsync - mtree_excludes_file="$(mktemp --tmpdir "${PROGNAME}.mtree-excludes.XXXXXX")" - temp_files+=("${mtree_excludes_file}") - grep -E "^([^/]|${line})" "${rsync_excludes_file}" | sed -e "s|^${line}|.|" > "${mtree_excludes_file}" - - mtree_file="/var/log/evobackup.$(basename "${line}").mtree" - temp_files+=("${mtree_file}") - - ${mtree_bin} -x -c -p "${line}" -X "${mtree_excludes_file}" > "${mtree_file}" - mtree_files+=("${mtree_file}") - done < "${rsync_includes_file}" - - if [ "${#mtree_files[@]}" -le 0 ]; then - log_error "SYNC_TASKS - ERROR: mtree didn't produce any file" - fi - - log "SYNC_TASKS - stop mtree (files: ${mtree_files[*]})" - else - log "SYNC_TASKS - skip mtree (missing)" - fi - else - log "SYNC_TASKS - skip mtree (disabled)" - fi - - rsync_bin=$(command -v rsync) - # Build the final Rsync command - - # Rsync main options - rsync_main_args=() - rsync_main_args+=(--archive) - rsync_main_args+=(--itemize-changes) - rsync_main_args+=(--quiet) - rsync_main_args+=(--stats) - rsync_main_args+=(--human-readable) - rsync_main_args+=(--relative) - rsync_main_args+=(--partial) - rsync_main_args+=(--delete) - rsync_main_args+=(--delete-excluded) - rsync_main_args+=(--force) - rsync_main_args+=(--ignore-errors) - rsync_main_args+=(--log-file "${RSYNC_LOGFILE}") - rsync_main_args+=(--rsh "ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'") - - # Rsync excludes - while read -r line ; do - rsync_main_args+=(--exclude "${line}") - done < "${rsync_excludes_file}" - - # Rsync local sources - # Start with default includes - # shellcheck disable=SC2206 - rsync_main_args+=(${rsync_default_includes[@]}) - # … and add custom includes - while read -r line ; do - rsync_main_args+=("${line}") - done < "${rsync_includes_file}" - - # Rsync remote destination - rsync_main_args+=("root@${SSH_SERVER}:/var/backup/") - - # … log it - log "SYNC_TASKS - Rsync main command : ${rsync_bin} ${rsync_main_args[*]}" - - # … execute it - ${rsync_bin} "${rsync_main_args[@]}" - - rsync_main_rc=$? - - # Copy last lines of rsync log to the main log - tail -n 30 "${RSYNC_LOGFILE}" >> "${LOGFILE}" - # Copy Rsync stats to special file - tail -n 30 "${RSYNC_LOGFILE}" | grep --invert-match --extended-regexp " [\<\>ch\.\*]\S{10} " > "${RSYNC_STATSFILE}" - - # We ignore rc=24 (vanished files) - if [ ${rsync_main_rc} -ne 0 ] && [ ${rsync_main_rc} -ne 24 ]; then - log_error "SYNC_TASKS - rsync returned an error ${rsync_main_rc}" "${LOGFILE}" - rc=${E_SYNCFAILED} - else - # Build the report Rsync command - rsync_report_args=() - # Rsync options - rsync_report_args+=(--rsh "ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'") - # Rsync local source - if [ "${#mtree_files[@]}" -gt 0 ]; then - # send mtree files if there is any - rsync_report_args+=("${mtree_files[@]}") - fi - if [ -f "${RSYNC_LOGFILE}" ]; then - # send rsync full log file if it exists - rsync_report_args+=("${RSYNC_LOGFILE}") - fi - if [ -f "${RSYNC_STATSFILE}" ]; then - # send rsync stats log file if it exists - rsync_report_args+=("${RSYNC_STATSFILE}") - fi - # Rsync remote destination - rsync_report_args+=("root@${SSH_SERVER}:/var/log/") - - # … log it - log "SYNC_TASKS - Rsync report command : ${rsync_bin} ${rsync_report_args[*]}" - - # … execute it - ${rsync_bin} "${rsync_report_args[@]}" - fi - - log "STOP SYNC_TASKS - server=${server}" -} - -# Output a message to the log file -log() { - msg="${1:-$(cat /dev/stdin)}" - pid=$$ - printf "[%s] %s[%s]: %s\\n" \ - "$(/bin/date +"${DATE_FORMAT}")" "${PROGNAME}" "${pid}" "${msg}" \ - >> "${LOGFILE}" -} -log_error() { - error_msg=${1} - error_file=${2:""} - - if [ -n "${error_file}" ] && [ -f "${error_file}" ]; then - printf "\n### %s\n" "${error_msg}" >&2 - # shellcheck disable=SC2046 - if [ $(wc -l "${error_file}") -gt 30 ]; then - printf "~~~{%s (tail -30)}\n" "${error_file}" >&2 - tail -n 30 "${error_file}" >&2 - else - printf "~~~{%s}\n" "${error_file}" >&2 - cat "${error_file}" >&2 - fi - printf "~~~\n" >&2 - - log "${error_msg}, check ${error_file}" - else - printf "\n### %s\n" "${error_msg}" >&2 - - log "${error_msg}" - fi - -} -# Remove all temporary file created during the execution -# shellcheck disable=SC2317 -clean_temp_files() { - # shellcheck disable=SC2086 - rm -f "${temp_files[@]}" -} -main() { - START_EPOCH=$(/bin/date +%s) - START_TIME=$(/bin/date +"%Y%m%d%H%M%S") - - log "START GLOBAL - VERSION=${VERSION} LOCAL_TASKS=${LOCAL_TASKS} SYNC_TASKS=${SYNC_TASKS}" - - # shellcheck disable=SC2174 - mkdir -p -m 700 "${LOCAL_BACKUP_DIR}" - - ERRORS_DIR="${LOCAL_BACKUP_DIR}/${PROGNAME}.errors-${START_TIME}" - # shellcheck disable=SC2174 - mkdir -p -m 700 "${ERRORS_DIR}" - - ## Force umask - umask 077 - - ## Initialize variable to store SSH connection errors - SERVERS_SSH_ERRORS="" - - ## Verify other evobackup process and kill if needed - if [ -e "${PIDFILE}" ]; then - pid=$(cat "${PIDFILE}") - # Does process still exist ? - if kill -0 "${pid}" 2> /dev/null; then - # Killing the childs of evobackup. - for ppid in $(pgrep -P "${pid}"); do - kill -9 "${ppid}"; - done - # Then kill the main PID. - kill -9 "${pid}" - printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\\n" >&2 - else - rm -f "${PIDFILE}" - fi - fi - echo "$$" > "${PIDFILE}" - - # Initialize a list of files to delete at exit - # Any file added to the list will also be deleted at exit - temp_files=("${PIDFILE}") - - trap clean_temp_files EXIT - - # Update canary to keep track of each run - update-evobackup-canary --who "${PROGNAME}" --file "${CANARY_FILE}" - - if [ "${LOCAL_TASKS}" = "1" ]; then - local_tasks - fi - - if [ "${SYNC_TASKS}" = "1" ]; then - sync_tasks - fi - - STOP_EPOCH=$(/bin/date +%s) - - if [ "${SYSTEM}" = "openbsd" ]; then - start_time=$(/bin/date -f "%s" -j "${START_EPOCH}" +"${DATE_FORMAT}") - stop_time=$(/bin/date -f "%s" -j "${STOP_EPOCH}" +"${DATE_FORMAT}") - else - start_time=$(/bin/date --date="@${START_EPOCH}" +"${DATE_FORMAT}") - stop_time=$(/bin/date --date="@${STOP_EPOCH}" +"${DATE_FORMAT}") - fi - duration=$(( STOP_EPOCH - START_EPOCH )) - - log "STOP GLOBAL - start='${start_time}' stop='${stop_time}' duration=${duration}s" - - tail -20 "${LOGFILE}" | mail -s "[info] EvoBackup - Client ${HOSTNAME}" ${MAIL} -} - -# set all programs to C language (english) -export LC_ALL=C - -# If expansion is attempted on an unset variable or parameter, the shell prints an -# error message, and, if not interactive, exits with a non-zero status. -set -u -# The pipeline's return status is the value of the last (rightmost) command -# to exit with a non-zero status, or zero if all commands exit successfully. -set -o pipefail - -# Default return-code (0 == succes) -rc=0 - -# Possible error codes -E_NOSRVAVAIL=21 # No server is available -E_SYNCFAILED=20 # Faild sync task -E_DUMPFAILED=10 # Faild dump task - -# explicit PATH -PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin - -# You can set "linux" or "bsd" manually or let it choose automatically -SYSTEM=$(uname | tr '[:upper:]' '[:lower:]') - -# Hostname reported in the mail subject -HOSTNAME=$(hostname) - -# Store pid in a file named after this program's name -PROGNAME=$(basename "$0") -PIDFILE="/var/run/${PROGNAME}.pid" - -# Customize the log path if you want multiple scripts to have separate log files -LOGFILE="/var/log/evobackup.log" -# Rsync complete log file for the current run -RSYNC_LOGFILE="/var/log/${PROGNAME}.rsync.log" -# Rsync stats for the current run -RSYNC_STATSFILE="/var/log/${PROGNAME}.rsync-stats.log" -# Canary file to update before executing tasks -CANARY_FILE="/zzz_evobackup_canary" - -DATE_FORMAT="%Y-%m-%d %H:%M:%S" - -# Should we fallback on other servers when the first one is unreachable? -SERVERS_FALLBACK=${SERVERS_FALLBACK:-1} -# timeout (in seconds) for SSH connections -SSH_CONNECT_TIMEOUT=${SSH_CONNECT_TIMEOUT:-90} - -# execute main function -main - -exit ${rc} +main \ No newline at end of file diff --git a/client/zzz_evobackup.sh b/client/zzz_evobackup.sh deleted file mode 100644 index 9afe3d2..0000000 --- a/client/zzz_evobackup.sh +++ /dev/null @@ -1,272 +0,0 @@ -#!/bin/bash -# -# Script Evobackup client -# See https://gitea.evolix.org/evolix/evobackup -# -# Authors: Evolix , -# Gregory Colpart , -# Romain Dessort , -# Benoit Série , -# Tristan Pilat , -# Victor Laborie , -# Jérémy Lecour -# and others. -# -# Licence: AGPLv3 - -####################################################################### -# -# You must configure the MAIL variable to receive notifications. -# -# There is some optional configuration that you can do -# at the end of this script. -# -# The library (usually installed at /usr/local/lib/evobackup/main.sh) -# also has many variables that you can override for fine-tuning. -# -####################################################################### - -# Email adress for notifications -MAIL=jdoe@example.com - -####################################################################### -# -# The "sync_tasks" function will be called by the main function. -# -# You can customize the variables: -# * "sync_name" (String) -# * "SERVERS" (Array of HOST:PORT) -# * "RSYNC_INCLUDES" (Array of paths to include) -# * "RSYNC_EXCLUDES" (Array of paths to exclude) -# -# The "sync" function can be called multiple times -# with a different set of variables. -# That way you can to sync to various destinations. -# -####################################################################### - -sync_tasks() { - - ########## System-only backup (to Evolix servers) ################# - - # Name your sync task, for logs - sync_name="evolix-system" - - # List of host/port for your sync task - # shellcheck disable=SC2034 - SERVERS=( - node0.backup.evolix.net:2234 - node1.backup.evolix.net:2234 - ) - - # What to include in your sync task - # Add or remove paths if you need - # shellcheck disable=SC2034 - RSYNC_INCLUDES=( - "${rsync_default_includes[@]}" - /etc - /root - /var - ) - - # What to exclude from your sync task - # Add or remove paths if you need - # shellcheck disable=SC2034 - RSYNC_EXCLUDES=( - "${rsync_default_excludes[@]}" - ) - - # Call the sync task - sync "${sync_name}" "SERVERS[@]" "RSYNC_INCLUDES[@]" "RSYNC_EXCLUDES[@]" - - - ########## Full backup (to client servers) ######################## - - # Name your sync task, for logs - sync_name="client-full" - - # List of host/port for your sync task - # shellcheck disable=SC2034 - SERVERS=( - client-backup00.evolix.net:2221 - client-backup01.evolix.net:2221 - ) - - # What to include in your sync task - # Add or remove paths if you need - # shellcheck disable=SC2034 - RSYNC_INCLUDES=( - "${rsync_default_includes[@]}" - /etc - /root - /var - /home - /srv - ) - - # What to exclude from your sync task - # Add or remove paths if you need - # shellcheck disable=SC2034 - RSYNC_EXCLUDES=( - "${rsync_default_excludes[@]}" - ) - - # Call the sync task - sync "${sync_name}" "SERVERS[@]" "RSYNC_INCLUDES[@]" "RSYNC_EXCLUDES[@]" - -} - -####################################################################### -# -# The "local_tasks" function will be called by the main function. -# -# You can call any available "dump_xxx" function -# (usually installed at /usr/local/lib/evobackup/dump.sh) -# -# You can also write some custom functions and call them. -# A "dump_custom" example is available further down. -# -####################################################################### - -local_tasks() { - - ########## OpenLDAP ############### - - ### dump_ldap - - ########## MySQL ################## - - # Dump all grants (permissions), config variables and schema of databases - ### dump_mysql_meta [--port=3306] - - # Dump all databases in a single compressed file - ### dump_mysql_global [--port=3306] [--masterdata] - - # Dump each database separately, in a compressed file - ### dump_mysql_per_base [--port=3306] - - # Dump multiples instances, each in a single compressed file - ### dump_mysql_instance [--port=3306] - - # Dump each table in schema/data files, for all databases - ### dump_mysql_tabs [--port=3306] [--user=foo] [--password=123456789] - - ########## PostgreSQL ############# - - # Dump all databases in a single file (compressed or not) - ### dump_postgresql_global - - # Dump a specific databse with only some tables, or all but some tables (must be configured) - ### dump_postgresql_filtered - - # Dump each database separately, in a compressed file - ### dump_postgresql_per_base - - ########## MongoDB ################ - - ### dump_mongodb [--user=foo] [--password=123456789] - - ########## Redis ################## - - # Copy data file for all instances - ### dump_redis [--instances=] - - ########## Elasticsearch ########## - - # Snapshot data for a single-node cluster - ### dump_elasticsearch_snapshot_singlenode [--protocol=http] [--host=localhost] [--port=9200] [--user=foo] [--password=123456789] [--repository=snaprepo] [--snapshot=snapshot.daily] - - # Snapshot data for a multi-node cluster - ### dump_elasticsearch_snapshot_multinode [--protocol=http] [--host=localhost] [--port=9200] [--user=foo] [--password=123456789] [--repository=snaprepo] [--snapshot=snapshot.daily] [--nfs-server=192.168.2.1] - - ########## RabbitMQ ############### - - ### dump_rabbitmq - - ########## MegaCli ################ - - # Copy RAID config - ### dump_megacli_config - - ########## Network ################ - - # Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) - ### dump_traceroute --targets=host_or_ip[,host_or_ip] - dump_traceroute --targets=8.8.8.8,www.evolix.fr,travaux.evolix.net - - ########## Server state ########### - - # Run dump-server-state to extract system information - ### dump-server-state [any dump-server-state option] - dump_server_state - - # Dump file access control lists - ### dump_facl - - # No-op, in case nothing is enabled - : -} - -# This is an example for a custom dump function -# Uncomment, customize and call it from the "local_tasks" function -### dump_custom() { -### # Set dump and errors directories and files -### local dump_dir="${LOCAL_BACKUP_DIR}/custom" -### local dump_file="${dump_dir}/dump.gz" -### local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") -### local error_file="${errors_dir}/dump.err" -### -### # Reset dump and errors directories -### rm -rf "${dump_dir}" "${errors_dir}" -### # shellcheck disable=SC2174 -### mkdir -p -m 700 "${dump_dir}" "${errors_dir}" -### -### # Log the start of the command -### log "LOCAL_TASKS - start ${dump_file}" -### -### # Execute your dump command -### # Send errors to the error file and the data to the dump file -### my-dump-command 2> "${error_file}" > "${dump_file}" -### -### # Check result and deal with potential errors -### local last_rc=$? -### # shellcheck disable=SC2086 -### if [ ${last_rc} -ne 0 ]; then -### log_error "LOCAL_TASKS - my-dump-command to ${dump_file} returned an error ${last_rc}" "${error_file}" -### GLOBAL_RC=${E_DUMPFAILED} -### else -### rm -f "${error_file}" -### fi -### -### # Log the end of the command -### log "LOCAL_TASKS - stop ${dump_file}" -### } - -########## Optional configuration ##################################### - -setup_custom() { - # If you set a value (like "linux", "openbsd"…) it will be used, - # Default: uname(1) in lowercase. - ### SYSTEM="linux" - - # If you set a value it will be used, - # Default: hostname(1). - ### HOSTNAME="example-host" - - # Email subect for notifications - ### MAIL_SUBJECT="[info] EvoBackup - Client ${HOSTNAME}" - - # No-op in case nothing is executed - : -} - -########## Libraries ################################################## - -# Change this to wherever you install the libraries -LIBDIR="/usr/local/lib/evobackup" - -source "${LIBDIR}/main.sh" - -########## Let's go! ################################################## - -main \ No newline at end of file