server > Introduce monitoring of inc creation time (#80)

Reviewed-on: #80

client/CHANGELOG.md

@@ -23,13 +23,69 @@ The **patch** part changes is incremented if multiple releases happen the same m
 
 ### Security
 
-## [24.04.1]
+## [25.01] - 2025-01-07
+
+### Fixed
+
+* dump_elasticsearch: Add an `\n` before HTTP status code
+* Fix compatibility with OpenBSD
+
+### Added
+
+* Add function for backup Linstor Database
+* Introducing munin-plugins for bkctld
+
+## [24.12] - 2022-12-18
+
+### Changed
+
+* client: the template defaults to full Evolix backup
+
+### Fixed
+
+* client: add exclude for 'lxc/*/rootfs/dev'
+
+## [24.11] - 2022-11-29
+
+### Fixed
+
+* dump_postgresql_per_base: look for correct variable to build connect options
+* dump_postgresql_per_base: return if postgresql directory is missing
+* dump_redis: choose all instances if none given
+* dump_redis: clean backup directory only if the Redis instance is valid
+* dump_redis: do not delete entire backup directory
+* dump_redis: use dynamic config for each instance
+
+## [24.07] - 2022-07-16
+
+### Changed
+
+* Allow everybody to "x" on LOCAL_BACKUP_DIR
+* dump/mysql.sh : give write permissions to mysql on tabs directories
+
+## [24.05.1] - 2022-05-14
+
+### Fixed
+
+* client: fix shell syntax error
+
+## [24.05] - 2022-05-02
+
+### Added
+
+* evobackupctl: update LIBDIR when copying the template
+
+### Changed
+
+* evobackupctl: simplify the program path retrieval
+
+## [24.04.1] - 2022-04-30
 
 ### Fixed
 
 * evobackupctl: quote ARGS variable for options parsing.
 
-## [24.04]
+## [24.04] - 2022-04-29
 
 ### Added
 
@@ -52,7 +108,7 @@ The **patch** part changes is incremented if multiple releases happen the same m
 
 * don't exit the whole program if a sync task can't be done
 
-## [22.12]
+## [22.12] - 2022-12-27
 
 ### Changed
 
@@ -77,6 +133,6 @@ The **patch** part changes is incremented if multiple releases happen the same m
 
 * Make start_time and stop_time compatible with OpenBSD
 
-## [22.03]
+## [22.03] - 2022-04-03
 
 Split client and server parts of the project

client/README.md

@@ -37,6 +37,8 @@ To install, copy these files :
 
 To update, simply overwrite them, since their content should (must?) not be customized locally.
 
+There is also an [evobackup-client](https://gitea.evolix.org/evolix/ansible-roles/src/branch/unstable/evobackup-client) Ansible role to do this (and some other stuff) automatically.
+
 ## Usage
 
 ### backup script

client/bin/evobackupctl

@@ -1,9 +1,8 @@
 #!/usr/bin/env bash
 
 # shellcheck disable=SC2155
-readonly PROGNAME=$(basename "${0}")
-# shellcheck disable=SC2155
-readonly PROGDIR=$(readlink -m "$(dirname "${0}")")
+readonly PROGPATH=$(readlink -f "${0}")
+readonly PROGNAME=$(basename "${PROGPATH}")
 # shellcheck disable=SC2124
 readonly ARGS=$@
 
@@ -36,6 +35,17 @@ Options
 END
 }
 
+get_server_ip_from_external() {
+    curl --connect-timeout 10 --ipv4 https://ifconfig.me 2> /dev/null
+}
+get_server_ip_from_internal() {
+    if [ "$(uname -s)" = "OpenBSD" ]; then
+        ifconfig egress | grep "inet " | head -1 | awk '{ print $2}'
+    else
+        hostname -I | awk '{ print $1}'
+    fi
+}
+
 jail_init_commands() {
     if [ ! -f /root/.ssh/id_ed25519.pub ]; then
         ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519 -N ''
@@ -44,11 +54,7 @@ jail_init_commands() {
 
     SSH_KEY=$(cat /root/.ssh/id_ed25519.pub)
     SERVER_NAME=$(hostname -s)
-    if [ "$(uname -s)" = "OpenBSD" ]; then
-        SERVER_IP=$(ifconfig egress | grep "inet " | head -1 | awk '{ print $2}')
-    else
-        SERVER_IP=$(curl -4 https://ifconfig.me 2> /dev/null || hostname -I | awk '{ print $1}')
-    fi
+    SERVER_IP=$(get_server_ip_from_external || get_server_ip_from_internal)
 
     echo "Copy-paste those lines on backup server(s) :"
     echo "----------"
@@ -78,10 +84,15 @@ copy_template() {
         if cp "${LIBDIR}/zzz_evobackup.sh" "${dest_path}"; then
             chmod 750 "${dest_path}"
 
-            sed -i "s|@COMMAND@|${PROGDIR}/${PROGNAME} ${ARGS}|" "${dest_path}"
-            sed -i "s|@DATE@|$(date --iso-8601=seconds)|" "${dest_path}"
+            # Insert metadata about the template
+            sed -i "s|@COMMAND@|${PROGPATH} ${ARGS}|" "${dest_path}"
+            # option "--iso-8601=seconds" does not exist on OpenBSD (and "%:z" neither)
+            sed -i "s|@DATE@|$(date +%FT%T%z)|" "${dest_path}"
             sed -i "s|@VERSION@|${VERSION}|" "${dest_path}"
 
+            # Make sure that the library directory is correct
+            sed -i "s|^LIBDIR=.\+|LIBDIR=\"${LIBDIR}\"|" "${dest_path}"
+
             printf "New evobackup script has been saved to '%s'.\n" "${dest_path}"
             printf "Remember to customize it (mail notifications, backup servers…).\n"
             exit 0

client/lib/dump/elasticsearch.sh

@@ -240,7 +240,7 @@ dump_elasticsearch() {
         connect_options+=(${option_others})
     fi
     # Add the http return code at the end of the output
-    connect_options+=(--write-out '%{http_code}\n')
+    connect_options+=(--write-out '\n%{http_code}\n')
     connect_options+=(--silent)
 
     declare -a dump_options

client/lib/dump/misc.sh

@@ -34,12 +34,13 @@ dump_ldap() {
 # Copy dump file of Redis instances
 #
 # Arguments:
-# --instances=[Integer] (default: all)
+# --instances=[String,String] (default: all)
 #######################################################################
 dump_redis() {
-    all_instances=$(find /var/lib/ -mindepth 1 -maxdepth 1 '(' -type d -o -type l ')' -name 'redis*')
+    all_instances=$( find /etc/redis*/redis.conf -print0 | xargs -0 -n 1 dirname  | xargs -n 1 basename | sort -h | xargs )
+
+    declare -a option_instances
 
-    local option_instances=""
     # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a
     while :; do
         case ${1:-''} in
@@ -87,51 +88,70 @@ dump_redis() {
 
         shift
     done
-    
+
+    # If no instance has been give, use every instance found
+    if [ -z "${option_instances[*]}" ]; then
+        read -a option_instances <<< "${all_instances}"
+    fi
+
     for instance in "${option_instances[@]}"; do
-        name=$(basename "${instance}")
-        local dump_dir="${LOCAL_BACKUP_DIR}/${name}"
-        local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") 
-        rm -rf "${dump_dir}" "${errors_dir}"
-        mkdir -p "${dump_dir}" "${errors_dir}"
-        # No need to change recursively, the top directory is enough
-        chmod 700 "${dump_dir}" "${errors_dir}"
+        # Look for the config file for this instance
+        config_file="/etc/${instance}/redis.conf"
+        if [ -n "${instance}" ] && [ -f "${config_file}" ]; then
+            # Parse the config for he data dir and dump file
+            local instance_data_dir
+            local instance_data_file
+            local instance_dump_path
 
-        if [ -f "${instance}/dump.rdb" ]; then
-            local error_file="${errors_dir}/${name}.err"
-            log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_dir}"
+            instance_data_dir=$( grep --extended-regexp "^\s*dir " "${config_file}" | cut -d ' ' -f 2 | tr -d "'" | tr -d '"' )
+            instance_data_file=$( grep --extended-regexp "^\s*dbfilename " "${config_file}" | cut -d ' ' -f 2 | tr -d "'" | tr -d '"' )
+            instance_dump_path="${instance_data_dir}/${instance_data_file}"
 
-            # Copy the Redis database
-            dump_cmd="cp -a ${instance}/dump.rdb ${dump_dir}/dump.rdb"
-            log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}"
-            ${dump_cmd} 2> "${error_file}"
+            if [ -n "${instance_dump_path}" ]; then
+                local dump_dir="${LOCAL_BACKUP_DIR}/${instance}"
+                local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") 
+                rm -rf "${dump_dir}" "${errors_dir}"
+                mkdir -p "${dump_dir}" "${errors_dir}"
+                # No need to change recursively, the top directory is enough
+                chmod 700 "${dump_dir}" "${errors_dir}"
 
-            local last_rc=$?
-            # shellcheck disable=SC2086
-            if [ ${last_rc} -ne 0 ]; then
-                log_error "LOCAL_TASKS - ${FUNCNAME[0]}: cp ${instance}/dump.rdb to ${dump_dir} returned an error ${last_rc}" "${error_file}"
-                GLOBAL_RC=${E_DUMPFAILED}
+                local error_file="${errors_dir}/${instance}.err"
+                log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_dir}"
+
+                # Copy the Redis database
+                dump_cmd="cp -a ${instance_dump_path} ${dump_dir}/${instance_data_file}"
+                log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}"
+                ${dump_cmd} 2> "${error_file}"
+
+                local last_rc=$?
+                # shellcheck disable=SC2086
+                if [ ${last_rc} -ne 0 ]; then
+                    log_error "LOCAL_TASKS - ${FUNCNAME[0]}: cp ${instance_dump_path} to ${dump_dir} returned an error ${last_rc}" "${error_file}"
+                    GLOBAL_RC=${E_DUMPFAILED}
+                else
+                    rm -f "${error_file}"
+                fi
+
+                # Compress the Redis database
+                dump_cmd="gzip ${dump_dir}/${instance_data_file}"
+                log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}"
+                ${dump_cmd}
+
+                local last_rc=$?
+                # shellcheck disable=SC2086
+                if [ ${last_rc} -ne 0 ]; then
+                    log_error "LOCAL_TASKS - ${FUNCNAME[0]}: gzip ${dump_dir}/${instance_data_file} returned an error ${last_rc}" "${error_file}"
+                    GLOBAL_RC=${E_DUMPFAILED}
+                else
+                    rm -f "${error_file}"
+                fi
+
+                log "LOCAL_TASKS - ${FUNCNAME[0]}: stop  ${dump_dir}"
             else
-                rm -f "${error_file}"
+                log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '${dump_file}' not found."
             fi
-
-            # Compress the Redis database
-            dump_cmd="gzip ${dump_dir}/dump.rdb"
-            log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}"
-            ${dump_cmd}
-
-            local last_rc=$?
-            # shellcheck disable=SC2086
-            if [ ${last_rc} -ne 0 ]; then
-                log_error "LOCAL_TASKS - ${FUNCNAME[0]}: gzip ${dump_dir}/dump.rdb returned an error ${last_rc}" "${error_file}"
-                GLOBAL_RC=${E_DUMPFAILED}
-            else
-                rm -f "${error_file}"
-            fi
-
-            log "LOCAL_TASKS - ${FUNCNAME[0]}: stop  ${dump_dir}"
         else
-            log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '${instance}/dump.rdb' not found."
+            log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '${config_file}' not found."
         fi
     done
 }
@@ -371,7 +391,7 @@ dump_raid_config() {
 # --targets=[IP,HOST] (default: <none>)
 #######################################################################
 dump_traceroute() {
-    local option_targets=""
+    declare -a option_targets
 
     # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a
     while :; do
@@ -557,3 +577,70 @@ dump_facl() {
 
     log "LOCAL_TASKS - ${FUNCNAME[0]}: stop  ${dump_dir}"
 }
+#######################################################################
+# Dump Linstor Database
+# 
+# Arguments: <none>
+#######################################################################
+dump_linstordb() {
+    # Set dump and errors directories and files
+    local dump_dir="${LOCAL_BACKUP_DIR}/linstor"
+    local dump_file="${dump_dir}/dump_linstor_db"
+    local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") 
+    local error_file="${errors_dir}/dump.err"
+
+    # Reset dump and errors directories
+    rm -rf "${dump_dir}" "${errors_dir}"
+    # shellcheck disable=SC2174
+    mkdir -p -m 700 "${dump_dir}" "${errors_dir}"
+
+    # Log the start of the function
+    log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}"
+        
+    # Stop linstor-controller process before backup
+    linstorctrl_stop="/usr/bin/systemctl stop linstor-controller.service"
+    log "LOCAL_TASKS - ${FUNCNAME[0]}: ${linstorctrl_stop}"
+        
+    # Execute stop linstor-controller
+    ${linstorctrl_stop}
+    local last_rc=$?
+    if [ "${last_rc}" -ne "0" ]; then
+        log_error "LOCAL_TASKS - ${FUNCNAME[0]}: linstor stopping service failed"
+        GLOBAL_RC=${E_DUMPFAILED}
+    else
+        # Prepare the dump command (errors go to the error file and the data to the dump file)
+        dump_cmd="/usr/share/linstor-server/bin/linstor-database export-db ${dump_file}"
+        log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}"
+
+        # Execute the dump command
+        ${dump_cmd} 2> ${error_file}
+
+        # Check result and deal with potential errors
+        local last_rc=$?
+        # shellcheck disable=SC2086
+        if [ ${last_rc} -ne 0 ]; then
+            log_error "LOCAL_TASKS - ${FUNCNAME[0]}: linstor database export to ${dump_file} returned an error ${last_rc}" "${error_file}"
+            GLOBAL_RC=${E_DUMPFAILED}
+        else
+            rm -f "${error_file}"
+        fi
+
+        # Start linstor-controller process after backup
+        linstorctrl_start="/usr/bin/systemctl start linstor-controller.service"
+        log "LOCAL_TASKS - ${FUNCNAME[0]}: ${linstorctrl_start}"
+
+        # Execute start linstor-controller
+        ${linstorctrl_start}
+
+        # Check result and deal with potential errors
+        local last_rc=$?
+        # shellcheck disable=SC2086
+        if [ ${last_rc} -ne 0 ]; then
+            log_error "LOCAL_TASKS - ${FUNCNAME[0]}: linstor starting service failed"
+            GLOBAL_RC=${E_DUMPFAILED}
+        fi
+    fi
+
+    # Log the end of the function
+    log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}"
+}

client/lib/dump/mysql.sh

@@ -504,6 +504,7 @@ dump_mysql_grants() {
 # --dump-label=[String] (default: "default")
 #   used as suffix of the dump dir to differenciate multiple instances
 # --compress=<gzip|pigz|bzip2|xz|none> (default: "gzip")
+# --dump-slave=[1|2]
 # Other options after -- are passed as-is to mysqldump
 #######################################################################
 dump_mysql_global() {
@@ -518,6 +519,7 @@ dump_mysql_global() {
     local option_dump_label=""
     local option_compress=""
     local option_others=""
+    local option_dump_slave=""
 
     # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a
     while :; do
@@ -696,6 +698,37 @@ dump_mysql_global() {
                 log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument."
                 exit 1
                 ;;
+            --dump-slave)
+                # dump-slave options, require value "1" or "2"
+                if [ "$2" = "1" ]; then
+                    option_dump_slave="--dump-slave=${2}"
+                    shift
+                elif [ "$2" = "2" ]; then
+                    option_dump_slave="--dump-slave=${2}"
+                    shift
+                else
+                    log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-slave' requires a non-empty option argument is \"1\" or \"2\"."
+                    exit 1
+                fi
+                ;;
+            --dump-slave=?*)
+                value="${1#*=}"
+                if [ ${value} = "1" ]; then
+                    option_dump_slave="${1}"
+                    shift
+                elif [ ${value} = "2" ]; then
+                    option_dump_slave="${1}"
+                    shift
+                else
+                    log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-slave' require value \"1\" or \"2\"."
+                    exit 1
+                fi
+                ;;
+            --dump-slave=)
+                # dump-slave options, without value
+                log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-slave' requires a non-empty option argument, value is \"1\" or \"2\"."
+                exit 1
+                ;;
             --)
                 # End of all options.
                 shift
@@ -800,6 +833,9 @@ dump_mysql_global() {
     if [ -n "${option_masterdata}" ]; then
         dump_options+=("${option_masterdata}")
     fi
+    if [ -n "${option_dump_slave}" ]; then
+        dump_options+=("${option_dump_slave}")
+    fi
     if [ -n "${option_others}" ]; then
         # word splitting is deliberate here
         # shellcheck disable=SC2206
@@ -1510,8 +1546,8 @@ dump_mysql_tabs() {
         rm -rf "${dump_dir}" "${errors_dir}"
         mkdir -p "${dump_dir}" "${errors_dir}"
         # No need to change recursively, the top directory is enough
-        chmod 700 "${dump_dir}" "${errors_dir}"
-        chown -RL mysql "${dump_dir}"
+        chmod 750 "$(dirname "${dump_dir}")" "${errors_dir}"
+        chown -RL mysql:mysql "$(dirname "${dump_dir}")"
 
         local error_file="${errors_dir}.err"
         log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_dir}"

client/lib/dump/postgresql.sh

@@ -8,12 +8,15 @@
 # --dump-label=[String] (default: "default")
 #   used as suffix of the dump dir to differenciate multiple instances
 # --compress=<gzip|pigz|bzip2|xz|none> (default: "gzip")
+# --host=<ip|socket_dir> (default: "/var/lib/postgresql")
 # Other options after -- are passed as-is to pg_dump
 #######################################################################
 dump_postgresql_global() {
     local option_dump_label=""
     local option_compress=""
     local option_others=""
+    local option_host=""
+    local option_port=""
 
     # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a
     while :; do
@@ -56,6 +59,44 @@ dump_postgresql_global() {
                 log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument."
                 exit 1
                 ;;
+            --host)
+                # host options, with value separated by space
+                if [ -n "$2" ]; then
+                    option_host="${2}"
+                    shift
+                else
+                    log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--host' requires a non-empty option argument."
+                    exit 1
+                fi
+                ;;
+            --host=?*)
+                # host options, with value separated by =
+                option_host="${1#*=}"
+                ;;
+            --host=)
+                # host options, without value
+                log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--host' requires a non-empty option argument."
+                exit 1
+                ;;
+            --port)
+                # port options, with value separated by space
+                if [ -n "$2" ]; then
+                    option_port="${2}"
+                    shift
+                else
+                    log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument."
+                    exit 1
+                fi
+                ;;
+            --port=?*)
+                # port options, with value separated by =
+                option_port="${1#*=}"
+                ;;
+            --port=)
+                # port options, without value
+                log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument."
+                exit 1
+                ;;
             --)
                 # End of all options.
                 shift
@@ -99,15 +140,7 @@ dump_postgresql_global() {
     esac
 
     if [ -z "${option_dump_label}" ]; then
-        if [ -n "${option_defaults_group_suffix}" ]; then
-            option_dump_label="${option_defaults_group_suffix}"
-        elif [ -n "${option_port}" ]; then
-            option_dump_label="${option_port}"
-        elif [ -n "${option_socket}" ]; then
-            option_dump_label=$(path_to_str "${option_socket}")
-        else
-            option_dump_label="default"
-        fi
+        option_dump_label="default"
     fi
 
     local dump_dir="${LOCAL_BACKUP_DIR}/postgresql-${option_dump_label}-global"
@@ -124,6 +157,15 @@ dump_postgresql_global() {
 
     declare -a dump_options
     dump_options=()
+    if [ -n "${option_host}" ]; then
+       dump_options+=(--host ${option_host})
+       if [ -n "${option_port}" ]; then
+          dump_options+=(--port ${option_port})
+       else
+          dump_options+=(--port 5432)
+       fi
+    fi
+
     if [ -n "${option_others}" ]; then
         # word splitting is deliberate here
         # shellcheck disable=SC2206
@@ -132,7 +174,7 @@ dump_postgresql_global() {
     
     dump_cmd="(sudo -u postgres pg_dumpall ${dump_options[*]}) 2> ${error_file} | ${compress_cmd} > ${dump_file}"
     log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}"
-    ${dump_cmd}
+    eval "${dump_cmd}"
 
     local last_rc=$?
     # shellcheck disable=SC2086
@@ -166,6 +208,8 @@ dump_postgresql_per_base() {
     local option_dump_label=""
     local option_compress=""
     local option_others=""
+    local option_host=""
+    local option_port=""
 
     # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a
     while :; do
@@ -208,6 +252,44 @@ dump_postgresql_per_base() {
                 log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument."
                 exit 1
                 ;;
+            --host)
+                # host options, with value separated by space
+                if [ -n "$2" ]; then
+                    option_host="${2}"
+                    shift
+                else
+                    log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--host' requires a non-empty option argument."
+                    exit 1
+                fi
+                ;;
+            --host=?*)
+                # host options, with value separated by =
+                option_host="${1#*=}"
+                ;;
+            --host=)
+                # host options, without value
+                log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--host' requires a non-empty option argument."
+                exit 1
+                ;;
+            --port)
+                # port options, with value separated by space
+                if [ -n "$2" ]; then
+                    option_port="${2}"
+                    shift
+                else
+                    log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument."
+                    exit 1
+                fi
+                ;;
+            --port=?*)
+                # port options, with value separated by =
+                option_port="${1#*=}"
+                ;;
+            --port=)
+                # port options, without value
+                log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument."
+                exit 1
+                ;;
             --)
                 # End of all options.
                 shift
@@ -251,15 +333,7 @@ dump_postgresql_per_base() {
     esac
 
     if [ -z "${option_dump_label}" ]; then
-        if [ -n "${option_defaults_group_suffix}" ]; then
-            option_dump_label="${option_defaults_group_suffix}"
-        elif [ -n "${option_port}" ]; then
-            option_dump_label="${option_port}"
-        elif [ -n "${option_socket}" ]; then
-            option_dump_label=$(path_to_str "${option_socket}")
-        else
-            option_dump_label="default"
-        fi
+        option_dump_label="default"
     fi
 
     local dump_dir="${LOCAL_BACKUP_DIR}/postgresql-${option_dump_label}-per-base"
@@ -271,8 +345,21 @@ dump_postgresql_per_base() {
 
     (
         # shellcheck disable=SC2164
-        cd /var/lib/postgresql
-        databases=$(sudo -u postgres psql -U postgres -lt | awk -F \| '{print $1}' | grep -v "template.*")
+        
+        declare -a connect_options
+        if [ -n "${option_host}" ]; then
+            connect_options+=(--host ${option_host})
+            if [ -n "${option_port}" ]; then
+                connect_options+=(--port ${option_port})
+            else
+                connect_options+=(--port 5432)
+            fi
+        fi
+
+        cd /var/lib/postgresql || { log "LOCAL_TASKS - ${FUNCNAME[0]}: /var/lib/postgresql not found"; return; }
+    
+        databases=$(sudo -u postgres psql -U postgres ${connect_options[*]} -lt | awk -F \| '{print $1}' | grep -v "template.*")
+
         for database in ${databases} ; do
             local error_file="${errors_dir}/${database}.err"
             local dump_file="${dump_dir}/${database}.sql${dump_ext}"
@@ -283,6 +370,16 @@ dump_postgresql_per_base() {
             dump_options+=(--create)
             dump_options+=(-U postgres)
             dump_options+=(-d "${database}")
+
+            if [ -n "${option_host}" ]; then
+                dump_options+=(--host ${option_host})
+                if [ -n "${option_port}" ]; then
+                    dump_options+=(--port ${option_port})
+                else
+                    dump_options+=(--port 5432)
+                fi
+            fi
+
             if [ -n "${option_others}" ]; then
                 # word splitting is deliberate here
                 # shellcheck disable=SC2206
@@ -291,7 +388,7 @@ dump_postgresql_per_base() {
 
             dump_cmd="(sudo -u postgres /usr/bin/pg_dump ${dump_options[*]}) 2> ${error_file} | ${compress_cmd} > ${dump_file}"
             log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}"
-            ${dump_cmd}
+            eval "${dump_cmd}"
 
             local last_rc=$?
             # shellcheck disable=SC2086

client/lib/main.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # shellcheck disable=SC2034,SC2317
 
-readonly VERSION="24.04.1"
+readonly VERSION="25.01"
 
 # set all programs to C language (english)
 export LC_ALL=C
@@ -133,6 +133,7 @@ sync_tasks_wrapper() {
         'lxc/*/rootfs/var/run'
         'lxc/*/rootfs/var/state'
         'lxc/*/rootfs/var/tmp'
+        'lxc/*/rootfs/dev'
         /home/mysqltmp
     )
     readonly rsync_default_excludes
@@ -378,7 +379,7 @@ setup() {
 
     : "${LOCAL_BACKUP_DIR:="/home/backup"}"
     # shellcheck disable=SC2174
-    mkdir -p -m 700 "${LOCAL_BACKUP_DIR}"
+    mkdir -p -m 711 "${LOCAL_BACKUP_DIR}"
 
     : "${ERRORS_DIR:="${LOCAL_BACKUP_DIR}/${PROGNAME}.errors-${START_TIME}"}"
     # shellcheck disable=SC2174

client/lib/utilities.sh

@@ -105,7 +105,7 @@ test_server() {
 pick_server() {
     local -i increment=${1:-0}
     local -i list_length=${#SERVERS[@]}
-    local sync_name=${2:""}
+    local sync_name=${2:-""}
 
     if (( increment >= list_length )); then
         # We've reached the end of the list

client/lib/zzz_evobackup.sh

@@ -45,9 +45,9 @@ MAIL=__NOTIFICATION_MAIL__
 # shellcheck disable=SC2034
 sync_tasks() {
 
-    ########## System-only backup (to Evolix servers) #################
+    ########## Backup to Evolix servers ###############################
 
-    SYNC_NAME="evolix-system"
+    SYNC_NAME="evolix"
     SERVERS=(
         __SRV0_HOST__:__SRV0_PORT__
         __SRV1_HOST__:__SRV1_PORT__
@@ -57,6 +57,8 @@ sync_tasks() {
         /etc
         /root
         /var
+        /home
+        /srv
     )
     RSYNC_EXCLUDES=(
         "${rsync_default_excludes[@]}"
@@ -64,9 +66,9 @@ sync_tasks() {
     sync "${SYNC_NAME}" "SERVERS[@]" "RSYNC_INCLUDES[@]" "RSYNC_EXCLUDES[@]"
 
 
-    ########## Full backup (to client servers) ########################
+    ########## Backup to client servers ###############################
 
-    ### SYNC_NAME="client-full"
+    ### SYNC_NAME="client"
     ### SERVERS=(
     ###     client-backup00.evolix.net:2221
     ###     client-backup01.evolix.net:2221
@@ -219,7 +221,9 @@ local_tasks() {
 
     # Copy data file for all instances
     #
-    ### dump_redis [--instances=<all|instance1|instance2>]
+    # The instance name is the config directory name
+    # eg. "redis" for "/etc/redis/", "redis-foo" for "/etc/redis-foo"
+    ### dump_redis [--instances=<all|instance1,instance2>]
 
     ########## Elasticsearch ##########
 

server/CHANGELOG.md

@@ -13,18 +13,100 @@ The **patch** part changes is incremented if multiple releases happen the same m
 
 ### Added
 
+* bkctld-check-setup: Check how many incs operation are running (Critical if >=2)
+* bkctld-check-setup: Check if inc and rm operations are running simultaneously (Warning if yes)
+* bkctld-check-setup: Check if inc creation time (in the last 10 days) is bellow given thresholds
+* bkctld-check-setup: Check if there is more than 1 unfinished inc operation
+* bkctld-inc: Save inc creation time
+* Directory /var/lib/bkctld to store internal bkctld informations
+* Munin plugins for bkctld [#64](https://gitea.evolix.org/evolix/evobackup/pulls/64)
+
 ### Changed
 
+* bkctld-inc: Add log message at begining/end of operations (with the inc name)
+* bkctld-inc: Inverted log message priorities (progre