From 1c12827c9cb7153bffa624f3f89cd3c607fc8cc5 Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Fri, 30 Aug 2019 14:43:52 -0400
Subject: [PATCH 01/17] Added evobackup-client role

---
 CHANGELOG.md                                  |   1 +
 evobackup-client/README.md                    |  24 ++
 evobackup-client/defaults/main.yml            |  13 +
 evobackup-client/handlers/main.yml            |   5 +
 evobackup-client/tasks/main.yml               |  21 +
 evobackup-client/tasks/open_ssh_ports.yml     |  31 ++
 evobackup-client/tasks/ssh_key.yml            |  20 +
 evobackup-client/tasks/upload_scripts.yml     |  16 +
 evobackup-client/tasks/verify_ssh.yml         |  11 +
 .../templates/zzz_evobackup.default.sh.j2     | 365 ++++++++++++++++++
 10 files changed, 507 insertions(+)
 create mode 100644 evobackup-client/README.md
 create mode 100644 evobackup-client/defaults/main.yml
 create mode 100644 evobackup-client/handlers/main.yml
 create mode 100644 evobackup-client/tasks/main.yml
 create mode 100644 evobackup-client/tasks/open_ssh_ports.yml
 create mode 100644 evobackup-client/tasks/ssh_key.yml
 create mode 100644 evobackup-client/tasks/upload_scripts.yml
 create mode 100644 evobackup-client/tasks/verify_ssh.yml
 create mode 100644 evobackup-client/templates/zzz_evobackup.default.sh.j2

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 85161d0d..1491d18e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,7 @@ The **patch** part changes incrementally at each release.
 * packweb-apache: Deploy opcache.php to give some insights on PHP's opcache status
 * webapps/evoadmin-web Overload templates if needed
 * redis: max clients is configurable
+* evobackup-client role to configure a machine for backups with bkctld(8)
 
 ### Changed
 * elasticsearch: listen on local interface only by default
diff --git a/evobackup-client/README.md b/evobackup-client/README.md
new file mode 100644
index 00000000..99f9e2dd
--- /dev/null
+++ b/evobackup-client/README.md
@@ -0,0 +1,24 @@
+# evobackup-client
+
+Allows the configuration of backups to a pair of bkctld(8) hosts.
+
+The backup hosts in use need to be defined in evobackup-client___hosts
+and the bkctld jail ssh port has to be defined in
+evobackup-client___ssh_port before running it.
+
+The default zzz_evobackup.sh configures a system backup, but the
+template can be overriden to configure a full backup instead. If
+you change the variables in defaults/main.yml you can easily run
+this again and configure backups to a second set of bkctld(8) hosts.
+
+Do not forget to set the evobackup-client___mail variable to an
+email adress you control.
+
+You can add this example to an installation playbook to create the
+ssh key without running the rest of the role.
+
+~~~
+  post_tasks:
+    - include_role:
+        name: evobackup-client tasks_from: ssh_key.yml
+~~~
diff --git a/evobackup-client/defaults/main.yml b/evobackup-client/defaults/main.yml
new file mode 100644
index 00000000..576bde1d
--- /dev/null
+++ b/evobackup-client/defaults/main.yml
@@ -0,0 +1,13 @@
+---
+evobackup-client__root_key_path: "/root/.ssh/evobackup_id"
+evobackup-client___cron_path: "/etc/cron.daily/zzz_evobackup"
+evobackup-client___cron_template_name: "zzz_evobackup"
+evobackup-client___mail: null
+evobackup-client__pid_path: "/var/run/evobackup.pid"
+evobackup-client___log_path: "/var/log/evobackup.log"
+evobackup-client__backup_path: "/home/backup"
+evobackup-client___ssh_port: null
+evobackup-client___hosts: null
+#  - name: "backups.example.org"
+#    ip: "xxx.xxx.xxx.xxx"
+#    fingerprint: "ecdsa-sha2-nistp256 ..."
diff --git a/evobackup-client/handlers/main.yml b/evobackup-client/handlers/main.yml
new file mode 100644
index 00000000..5a0c6bfd
--- /dev/null
+++ b/evobackup-client/handlers/main.yml
@@ -0,0 +1,5 @@
+---
+- name: restart minifirewall
+  service:
+    name: minifirewall
+    state: restarted
diff --git a/evobackup-client/tasks/main.yml b/evobackup-client/tasks/main.yml
new file mode 100644
index 00000000..9b25f2bf
--- /dev/null
+++ b/evobackup-client/tasks/main.yml
@@ -0,0 +1,21 @@
+---
+
+- include: "ssh_key.yml"
+  tags:
+    - evobackup-client
+    - evobackup-client-backup-ssh_key
+
+- include: "upload_scripts.yml"
+  tags:
+    - evobackup-client
+    - evobackup-client-backup-scripts
+
+- include: "open_ssh_ports.yml"
+  tags:
+    - evobackup-client
+    - evobackup-client-backup-firewall
+
+- include: "verify_ssh.yml"
+  tags:
+    - evobackup-client
+    - evobackup-client-backup-hosts
diff --git a/evobackup-client/tasks/open_ssh_ports.yml b/evobackup-client/tasks/open_ssh_ports.yml
new file mode 100644
index 00000000..9cc83a3a
--- /dev/null
+++ b/evobackup-client/tasks/open_ssh_ports.yml
@@ -0,0 +1,31 @@
+---
+
+- name: Is there a Minifirewall ?
+  stat:
+    path: /etc/default/minifirewall
+  register: evobackup-client__minifirewall
+  tags:
+    - evobackup-client
+    - evobackup-client-backup-firewall
+
+- name: backup ssh port
+  blockinfile:
+    dest: /etc/default/minifirewall
+    marker: "# evobackup ssh port"
+    block: "evobackup_port={{ evobackup-client___ssh_port }}"
+  when: evobackup-client__minifirewall.stat.exists
+  tags:
+    - evobackup-client
+    - evobackup-client-backup-firewall
+
+- name: Add backup SSH port in /etc/default/minifirewall
+  blockinfile:
+    dest: /etc/default/minifirewall
+    marker: "# {{ item.name }}"
+    block: |
+      /sbin/iptables -A INPUT -p tcp --sport {{ evobackup-client___ssh_port }} --dport 1024:65535 -s {{ item.ip }} -m state --state ESTABLISHED,RELATED -j ACCEPT
+  with_items: "{{ evobackup-client___hosts }}"
+  when: evobackup-client__minifirewall.stat.exists
+  tags:
+    - evobackup-client
+    - evobackup-client-backup-firewall
diff --git a/evobackup-client/tasks/ssh_key.yml b/evobackup-client/tasks/ssh_key.yml
new file mode 100644
index 00000000..6322504f
--- /dev/null
+++ b/evobackup-client/tasks/ssh_key.yml
@@ -0,0 +1,20 @@
+---
+
+- name: Create SSH key
+  user:
+    name: root
+    generate_ssh_key: true
+    ssh_key_file: "{{ evobackup-client__root_key_path }}"
+    ssh_key_type: rsa
+  register: evobackup-client__root_key
+  tags:
+    - evobackup-client
+    - evobackup-client-backup-ssh_key
+
+- name: Print SSH key
+  debug:
+    var: evobackup-client__root_key.ssh_public_key
+  when: evobackup-client__root_key.ssh_public_key is defined
+  tags:
+    - evobackup-client
+    - evobackup-client-backup-ssh_key
diff --git a/evobackup-client/tasks/upload_scripts.yml b/evobackup-client/tasks/upload_scripts.yml
new file mode 100644
index 00000000..77646804
--- /dev/null
+++ b/evobackup-client/tasks/upload_scripts.yml
@@ -0,0 +1,16 @@
+---
+
+- name: Upload evobackup script
+  template:
+    src: "{{ item }}"
+    dest: "{{ evobackup-client___cron_path }}"
+    force: true
+    mode: 0755
+  with_first_found:
+    - "templates/evobackup-client/{{ evobackup-client___cron_template_name }}.{{ inventory_hostname }}.sh.j2"
+    - "templates/evobackup-client/{{ evobackup-client___cron_template_name }}.{{ host_group }}.sh.j2"
+    - "templates/evobackup-client/{{ evobackup-client___cron_template_name }}.sh.j2"
+    - "zzz_evobackup.default.sh.j2"
+  tags:
+    - evobackup-client
+    - evobackup-client-backup-scripts
diff --git a/evobackup-client/tasks/verify_ssh.yml b/evobackup-client/tasks/verify_ssh.yml
new file mode 100644
index 00000000..af4e24a9
--- /dev/null
+++ b/evobackup-client/tasks/verify_ssh.yml
@@ -0,0 +1,11 @@
+---
+
+- name: Verify evolix backup servers
+  known_hosts:
+    path: /root/.ssh/known_hosts
+    name: "[{{ item.name }}]:{{ evobackup-client___ssh_port }}"
+    key: "[{{ item.name }}]:{{ evobackup-client___ssh_port }} {{ item.fingerprint }}"
+  with_list: "{{ evobackup-client___hosts }}"
+  tags:
+    - evobackup-client
+    - evobackup-client-backup-hosts
diff --git a/evobackup-client/templates/zzz_evobackup.default.sh.j2 b/evobackup-client/templates/zzz_evobackup.default.sh.j2
new file mode 100644
index 00000000..9fd19974
--- /dev/null
+++ b/evobackup-client/templates/zzz_evobackup.default.sh.j2
@@ -0,0 +1,365 @@
+#!/bin/sh
+#
+# Script Evobackup client
+# See https://gitea.evolix.org/evolix/evobackup
+#
+# Author: Gregory Colpart <reg@evolix.fr>
+# Contributors:
+# Romain Dessort <rdessort@evolix.fr>
+# Benoît Série <bserie@evolix.fr>
+# Tristan Pilat <tpilat@evolix.fr>
+# Victor Laborie <vlaborie@evolix.fr>
+# Jérémy Lecour <jlecour@evolix.fr>
+#
+# Licence: AGPLv3
+#
+# /!\ DON'T FORGET TO SET "MAIL" and "SERVERS" VARIABLES
+
+##### Configuration ###################################################
+
+# email adress for notifications
+MAIL={{ evobackup-client___mail }}
+
+# list of hosts (hostname or IP) and SSH port for Rsync
+SERVERS="{% for host in evobackup-client___hosts %}{{ host.name }}:{{ evobackup-client___ssh_port }} {% endfor %}"
+
+# timeout (in seconds) for SSH connections
+SSH_CONNECT_TIMEOUT=30
+
+## We use /home/backup : feel free to use your own dir
+LOCAL_BACKUP_DIR="{{ evobackup-client__backup_path }}"
+
+# You can set "linux" or "bsd" manually or let it choose automatically
+SYSTEM=$(uname | tr '[:upper:]' '[:lower:]')
+
+# Change these 2 variables if you have more than one backup cron
+PIDFILE="{{ evobackup-client__pid_path }}"
+LOGFILE="{{ evobackup-client__log_path }}"
+
+## Enable/Disable tasks
+LOCAL_TASKS=${LOCAL_TASKS:-1}
+SYNC_TASKS=${SYNC_TASKS:-1}
+
+##### SETUP AND FUNCTIONS #############################################
+
+BEGINNING=$(/bin/date +"%d-%m-%Y ; %H:%M")
+
+# shellcheck disable=SC2174
+mkdir -p -m 700 ${LOCAL_BACKUP_DIR}
+
+PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin
+
+## lang = C for english outputs
+export LANGUAGE=C
+export LANG=C
+
+## Force umask
+umask 077
+
+## Initialize variable to store SSH connection errors
+SERVERS_SSH_ERRORS=""
+
+# Call test_server with "HOST:PORT" string
+# It will return with 0 if the server is reachable.
+# It will return with 1 and a message on stderr if not.
+test_server() {
+    item=$1
+    # split HOST and PORT from the input string
+    host=$(echo "${item}" | cut -d':' -f1)
+    port=$(echo "${item}" | cut -d':' -f2)
+
+    # Test if the server is accepting connections
+    ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" -i /root/.ssh/evobackup_id "${host}" -p "${port}" -t "exit"
+    # shellcheck disable=SC2181
+    if [ $? = 0 ]; then
+        # SSH connection is OK
+        return 0
+    else
+        # SSH connection failed
+        new_error=$(printf "Failed to connect to \`%s' within %s seconds" "${item}" "${SSH_CONNECT_TIMEOUT}")
+        SERVERS_SSH_ERRORS=$(printf "%s\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d')
+
+        return 1
+    fi
+}
+# Call pick_server with an optional positive integer to get the nth server in the list.
+pick_server() {
+    increment=${1:-0}
+    list_length=$(echo "${SERVERS}" | wc -w)
+
+    if [ "${increment}" -ge "${list_length}" ]; then
+        # We've reached the end of the list
+        new_error="No more server available"
+        SERVERS_SSH_ERRORS=$(printf "%s\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d')
+
+        printf "%s\n" "${SERVERS_SSH_ERRORS}" >&2
+        return 1
+    fi
+
+    # Extract the day of month, without leading 0 (which would give an octal based number)
+    today=$(date +%e)
+    # A salt is useful to randomize the starting point in the list
+    # but stay identical each time it's called for a server (based on hostname).
+    salt=$(hostname | cksum | cut -d' ' -f1)
+    # Pick an integer between 0 and the length of the SERVERS list
+    # It changes each day
+    item=$(( (today + salt + increment) % list_length ))
+    # cut starts counting fields at 1, not 0.
+    field=$(( item + 1 ))
+
+    echo "${SERVERS}" | cut -d' ' -f${field}
+}
+
+## Verify other evobackup process and kill if needed
+if [ -e "${PIDFILE}" ]; then
+    pid=$(cat "${PIDFILE}")
+    # Killing the childs of evobackup.
+    for ppid in $(ps h --ppid "${pid}" -o pid | tr -s '\n' ' '); do
+        kill -9 "${ppid}";
+    done
+    # Then kill the main PID.
+    kill -9 "${pid}"
+    printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\n" >&2
+fi
+echo "$$" > ${PIDFILE}
+# shellcheck disable=SC2064
+trap "rm -f ${PIDFILE}" EXIT
+
+##### LOCAL BACKUP ####################################################
+
+if [ "${LOCAL_TASKS}" = "1" ]; then
+    # You can comment or uncomment sections below to customize the backup
+
+    ## OpenLDAP : example with slapcat
+    # slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak
+
+    ### MySQL
+
+    ## example with global and compressed mysqldump
+    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
+    #  --opt --all-databases --force --events --hex-blob | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz
+
+    ## example with two dumps for each table (.sql/.txt) for all databases
+    # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
+    # | egrep -v "^(Database|information_schema|performance_schema|sys)" ); \
+    # do mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump ; \
+    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \
+    #  --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i; done
+
+    ## example with compressed SQL dump for each databases
+    # mkdir -p -m 700 /home/mysqldump/
+    # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
+    # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
+    #     mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i | gzip --best > /home/mysqldump/${i}.sql.gz
+    # done
+
+    ## example with *one* uncompressed SQL dump for *one* database (MYBASE)
+    # mkdir -p -m 700 /home/mysqldump/MYBASE
+    # chown -RL mysql /home/mysqldump/
+    # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -Q \
+    # --opt --events --hex-blob --skip-comments -T /home/mysqldump/MYBASE MYBASE
+
+    ## example with mysqlhotcopy
+    # mkdir -p -m 700 /home/mysqlhotcopy/
+    # mysqlhotcopy BASE /home/mysqlhotcopy/
+
+    ## example for multiples MySQL instances
+    # mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3)
+    # grep -E "^port\s*=\s*\d*" /etc/mysql/my.cnf |while read instance; do
+    #     instance=$(echo "$instance"|awk '{ print $3 }')
+    #     if [ "$instance" != "3306" ]
+    #     then
+    #         mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd > ${LOCAL_BACKUP_DIR}/mysql.$instance.bak
+    #     fi
+    # done
+
+    ### PostgreSQL
+
+    ## example with pg_dumpall (warning: you need space in ~postgres)
+    # su - postgres -c "pg_dumpall > ~/pg.dump.bak"
+    # mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/
+    ## another method with gzip directly piped
+    # cd /var/lib/postgresql
+    # sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
+    # cd - > /dev/null
+
+    ## example with all tables from MYBASE excepts TABLE1 and TABLE2
+    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE
+
+    ## example with only TABLE1 and TABLE2 from MYBASE
+    # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE
+
+    ## MongoDB : example with mongodump
+    ## don't forget to create use with read-only access
+    ## > use admin
+    ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } )
+    # test -d ${LOCAL_BACKUP_DIR}/mongodump/ && rm -rf ${LOCAL_BACKUP_DIR}/mongodump/
+    # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/
+    # mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/
+    # if [ $? -ne 0 ]; then
+    #   echo "Error with mongodump!"
+    # fi
+
+    ## Redis : example with copy .rdb file
+    # cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/
+
+    ## ElasticSearch, take a snapshot as a backup.
+    ## Warning: You need to have a path.repo configured.
+    ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes
+    # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log
+    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log
+    ## Clustered version here
+    ## It basically the same thing except that you need to check that NFS is mounted
+    # if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2'
+    # then
+    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log
+    #     curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log
+    # else
+    #     echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
+    # fi
+    ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
+    # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
+    #     curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
+    # done
+    # date=$(date +%F)
+    # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log
+
+    ## RabbitMQ : export config
+    #rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> $LOGFILE
+
+    # backup MegaCli config
+    #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null
+
+    ## Dump system and kernel versions
+    uname -a > ${LOCAL_BACKUP_DIR}/uname
+
+    ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls)
+    for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do
+        mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr}
+        traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1
+    done
+
+    ## Dump process with ps
+    ps auwwx >${LOCAL_BACKUP_DIR}/ps.out
+
+    if [ "${SYSTEM}" = "linux" ]; then
+        ## Dump network connections with ss
+        ss -taupen > ${LOCAL_BACKUP_DIR}/netstat.out
+
+        ## List Debian packages
+        dpkg -l > ${LOCAL_BACKUP_DIR}/packages
+        dpkg --get-selections > ${LOCAL_BACKUP_DIR}/packages.getselections
+        apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available
+
+        ## Dump MBR / table partitions
+        disks=$(find /dev/ -regex '/dev/\([sv]d[a-z]\|nvme[0-9]+n[0-9]+\)')
+        for disk in ${disks}; do
+            name=$(basename "${disk}")
+            dd if="${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${name}" bs=512 count=1 2>&1 | egrep -v "(records in|records out|512 bytes)"
+            fdisk -l "${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${name}"
+        done
+        cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions
+    else
+        ## Dump network connections with netstat
+        netstat -finet -atn > ${LOCAL_BACKUP_DIR}/netstat.out
+
+        ## List OpenBSD packages
+        pkg_info -m > ${LOCAL_BACKUP_DIR}/packages
+
+        ## Dump MBR / table partitions
+        ##disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions
+    fi
+
+fi
+
+##### REMOTE BACKUP ###################################################
+
+n=0
+server=""
+while :; do
+    server=$(pick_server "${n}")
+    test $? = 0 || exit 2
+
+    if test_server "${server}"; then
+        break
+    else
+        server=""
+        n=$(( n + 1 ))
+    fi
+done
+
+SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
+SSH_PORT=$(echo "${server}" | cut -d':' -f2)
+
+HOSTNAME=$(hostname)
+
+if [ "${SYSTEM}" = "linux" ]; then
+    rep="/bin /boot /lib /opt /sbin /usr"
+else
+    rep="/bsd /bin /sbin /usr"
+fi
+
+
+if [ "${SYNC_TASKS}" = "1" ]; then
+    # /!\ DO NOT USE COMMENTS in the rsync command /!\
+    # It breaks the command and destroys data, simply remove (or add) lines.
+
+    # Remote shell command
+    RSH_COMMAND="ssh -i /root/.ssh/evobackup_id -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'"
+
+    rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial \
+        --exclude "lost+found"                             \
+        --exclude ".nfs.*"                                 \
+        --exclude "/var/log"                               \
+        --exclude "/var/log/evobackup*"                    \
+        --exclude "/var/lib/mysql"                         \
+        --exclude "/var/lib/postgres"                      \
+        --exclude "/var/lib/postgresql"                    \
+        --exclude "/var/lib/sympa"                         \
+        --exclude "/var/lib/metche"                        \
+        --exclude "/var/run"                               \
+        --exclude "/var/lock"                              \
+        --exclude "/var/state"                             \
+        --exclude "/var/apt"                               \
+        --exclude "/var/cache"                             \
+        --exclude "/usr/src"                               \
+        --exclude "/usr/doc"                               \
+        --exclude "/usr/share/doc"                         \
+        --exclude "/usr/obj"                               \
+        --exclude "dev"                                    \
+        --exclude "/var/spool/postfix"                     \
+        --exclude "/var/lib/amavis/amavisd.sock"           \
+        --exclude "/var/lib/munin/*tmp*"                   \
+        --exclude "/var/lib/php5"                          \
+        --exclude "/var/spool/squid"                       \
+        --exclude "/var/lib/elasticsearch"                 \
+        --exclude "/var/lib/amavis/tmp"                    \
+        --exclude "/var/lib/clamav/*.tmp"                  \
+        --exclude "/home/mysqltmp"                         \
+        --exclude "/var/lib/php/sessions"                  \
+            ${rep}                                         \
+            /etc                                           \
+            /root                                          \
+            /var                                           \
+            /home/backup                                          \
+            /srv                                           \
+        -e "${RSH_COMMAND}"                                \
+        "root@${SSH_SERVER}:/var/backup/"                  \
+        | tail -30 >> $LOGFILE
+fi
+
+##### REPORTING #######################################################
+
+END=$(/bin/date +"%d-%m-%Y ; %H:%M")
+
+printf "EvoBackup - %s - START %s ON %s (LOCAL_TASKS=%s SYNC_TASKS=%s)\n" \
+       "${HOSTNAME}" "${BEGINNING}" "${SSH_SERVER}" "${LOCAL_TASKS}" "${SYNC_TASKS}" \
+       >> $LOGFILE
+
+printf "EvoBackup - %s - STOP %s ON %s (LOCAL_TASKS=%s SYNC_TASKS=%s)\n" \
+       "${HOSTNAME}" "${END}" "${SSH_SERVER}" "${LOCAL_TASKS}" "${SYNC_TASKS}" \
+       >> $LOGFILE
+
+tail -10 $LOGFILE |                                    \
+    mail -s "[info] EvoBackup - Client ${HOSTNAME}"    \
+    ${MAIL}

From 7d6a552c0996b5fb0e65b1dcaac929467a4b1b42 Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Tue, 3 Sep 2019 09:50:13 -0400
Subject: [PATCH 02/17] Apply README fixes from peer review

Removes unecessary precision.
---
 evobackup-client/README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/evobackup-client/README.md b/evobackup-client/README.md
index 99f9e2dd..68007fb9 100644
--- a/evobackup-client/README.md
+++ b/evobackup-client/README.md
@@ -1,17 +1,17 @@
 # evobackup-client
 
-Allows the configuration of backups to a pair of bkctld(8) hosts.
+Allows the configuration of backups to one or more remote filesystems.
 
-The backup hosts in use need to be defined in evobackup-client___hosts
+The backup hosts in use need to be defined in evobackup-client__hosts
 and the bkctld jail ssh port has to be defined in
-evobackup-client___ssh_port before running it.
+evobackup-client__ssh_port before running it.
 
 The default zzz_evobackup.sh configures a system backup, but the
 template can be overriden to configure a full backup instead. If
 you change the variables in defaults/main.yml you can easily run
-this again and configure backups to a second set of bkctld(8) hosts.
+this again and configure backups to a second set of hosts.
 
-Do not forget to set the evobackup-client___mail variable to an
+Do not forget to set the evobackup-client__mail variable to an
 email adress you control.
 
 You can add this example to an installation playbook to create the

From c773c901f22d12f39f8b930995f81aa2645b25d5 Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Tue, 3 Sep 2019 09:51:46 -0400
Subject: [PATCH 03/17] Fix to evobackup-client variable names

Somehow, an extra _ managed to slide itself in a few places.
---
 evobackup-client/defaults/main.yml                   | 12 ++++++------
 evobackup-client/tasks/open_ssh_ports.yml            |  6 +++---
 evobackup-client/tasks/upload_scripts.yml            |  8 ++++----
 evobackup-client/tasks/verify_ssh.yml                |  6 +++---
 .../templates/zzz_evobackup.default.sh.j2            |  4 ++--
 5 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/evobackup-client/defaults/main.yml b/evobackup-client/defaults/main.yml
index 576bde1d..e11de5c9 100644
--- a/evobackup-client/defaults/main.yml
+++ b/evobackup-client/defaults/main.yml
@@ -1,13 +1,13 @@
 ---
 evobackup-client__root_key_path: "/root/.ssh/evobackup_id"
-evobackup-client___cron_path: "/etc/cron.daily/zzz_evobackup"
-evobackup-client___cron_template_name: "zzz_evobackup"
-evobackup-client___mail: null
+evobackup-client__cron_path: "/etc/cron.daily/zzz_evobackup"
+evobackup-client__cron_template_name: "zzz_evobackup"
+evobackup-client__mail: null
 evobackup-client__pid_path: "/var/run/evobackup.pid"
-evobackup-client___log_path: "/var/log/evobackup.log"
+evobackup-client__log_path: "/var/log/evobackup.log"
 evobackup-client__backup_path: "/home/backup"
-evobackup-client___ssh_port: null
-evobackup-client___hosts: null
+evobackup-client__ssh_port: null
+evobackup-client__hosts: null
 #  - name: "backups.example.org"
 #    ip: "xxx.xxx.xxx.xxx"
 #    fingerprint: "ecdsa-sha2-nistp256 ..."
diff --git a/evobackup-client/tasks/open_ssh_ports.yml b/evobackup-client/tasks/open_ssh_ports.yml
index 9cc83a3a..7979d233 100644
--- a/evobackup-client/tasks/open_ssh_ports.yml
+++ b/evobackup-client/tasks/open_ssh_ports.yml
@@ -12,7 +12,7 @@
   blockinfile:
     dest: /etc/default/minifirewall
     marker: "# evobackup ssh port"
-    block: "evobackup_port={{ evobackup-client___ssh_port }}"
+    block: "evobackup_port={{ evobackup-client__ssh_port }}"
   when: evobackup-client__minifirewall.stat.exists
   tags:
     - evobackup-client
@@ -23,8 +23,8 @@
     dest: /etc/default/minifirewall
     marker: "# {{ item.name }}"
     block: |
-      /sbin/iptables -A INPUT -p tcp --sport {{ evobackup-client___ssh_port }} --dport 1024:65535 -s {{ item.ip }} -m state --state ESTABLISHED,RELATED -j ACCEPT
-  with_items: "{{ evobackup-client___hosts }}"
+      /sbin/iptables -A INPUT -p tcp --sport {{ evobackup-client__ssh_port }} --dport 1024:65535 -s {{ item.ip }} -m state --state ESTABLISHED,RELATED -j ACCEPT
+  with_items: "{{ evobackup-client__hosts }}"
   when: evobackup-client__minifirewall.stat.exists
   tags:
     - evobackup-client
diff --git a/evobackup-client/tasks/upload_scripts.yml b/evobackup-client/tasks/upload_scripts.yml
index 77646804..2bb94d6e 100644
--- a/evobackup-client/tasks/upload_scripts.yml
+++ b/evobackup-client/tasks/upload_scripts.yml
@@ -3,13 +3,13 @@
 - name: Upload evobackup script
   template:
     src: "{{ item }}"
-    dest: "{{ evobackup-client___cron_path }}"
+    dest: "{{ evobackup-client__cron_path }}"
     force: true
     mode: 0755
   with_first_found:
-    - "templates/evobackup-client/{{ evobackup-client___cron_template_name }}.{{ inventory_hostname }}.sh.j2"
-    - "templates/evobackup-client/{{ evobackup-client___cron_template_name }}.{{ host_group }}.sh.j2"
-    - "templates/evobackup-client/{{ evobackup-client___cron_template_name }}.sh.j2"
+    - "templates/evobackup-client/{{ evobackup-client__cron_template_name }}.{{ inventory_hostname }}.sh.j2"
+    - "templates/evobackup-client/{{ evobackup-client__cron_template_name }}.{{ host_group }}.sh.j2"
+    - "templates/evobackup-client/{{ evobackup-client__cron_template_name }}.sh.j2"
     - "zzz_evobackup.default.sh.j2"
   tags:
     - evobackup-client
diff --git a/evobackup-client/tasks/verify_ssh.yml b/evobackup-client/tasks/verify_ssh.yml
index af4e24a9..99dd6cb4 100644
--- a/evobackup-client/tasks/verify_ssh.yml
+++ b/evobackup-client/tasks/verify_ssh.yml
@@ -3,9 +3,9 @@
 - name: Verify evolix backup servers
   known_hosts:
     path: /root/.ssh/known_hosts
-    name: "[{{ item.name }}]:{{ evobackup-client___ssh_port }}"
-    key: "[{{ item.name }}]:{{ evobackup-client___ssh_port }} {{ item.fingerprint }}"
-  with_list: "{{ evobackup-client___hosts }}"
+    name: "[{{ item.name }}]:{{ evobackup-client__ssh_port }}"
+    key: "[{{ item.name }}]:{{ evobackup-client__ssh_port }} {{ item.fingerprint }}"
+  with_list: "{{ evobackup-client__hosts }}"
   tags:
     - evobackup-client
     - evobackup-client-backup-hosts
diff --git a/evobackup-client/templates/zzz_evobackup.default.sh.j2 b/evobackup-client/templates/zzz_evobackup.default.sh.j2
index 9fd19974..27346930 100644
--- a/evobackup-client/templates/zzz_evobackup.default.sh.j2
+++ b/evobackup-client/templates/zzz_evobackup.default.sh.j2
@@ -18,10 +18,10 @@
 ##### Configuration ###################################################
 
 # email adress for notifications
-MAIL={{ evobackup-client___mail }}
+MAIL={{ evobackup-client__mail }}
 
 # list of hosts (hostname or IP) and SSH port for Rsync
-SERVERS="{% for host in evobackup-client___hosts %}{{ host.name }}:{{ evobackup-client___ssh_port }} {% endfor %}"
+SERVERS="{% for host in evobackup-client__hosts %}{{ host.name }}:{{ evobackup-client__ssh_port }} {% endfor %}"
 
 # timeout (in seconds) for SSH connections
 SSH_CONNECT_TIMEOUT=30

From 0f7d9e9f2495f781c539d86a6873370a7dd2e6eb Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Tue, 3 Sep 2019 10:29:05 -0400
Subject: [PATCH 04/17] Allow heterogeneous ports

This makes it a bit more annoying to define default hosts for
multiple machines, but it's not that bad and it's better than not
being able to define heterogeous ports.
---
 evobackup-client/README.md                           |  7 +++----
 evobackup-client/defaults/main.yml                   |  2 +-
 evobackup-client/tasks/open_ssh_ports.yml            | 12 +-----------
 evobackup-client/tasks/verify_ssh.yml                |  4 ++--
 .../templates/zzz_evobackup.default.sh.j2            |  2 +-
 5 files changed, 8 insertions(+), 19 deletions(-)

diff --git a/evobackup-client/README.md b/evobackup-client/README.md
index 68007fb9..18ef132e 100644
--- a/evobackup-client/README.md
+++ b/evobackup-client/README.md
@@ -2,11 +2,10 @@
 
 Allows the configuration of backups to one or more remote filesystems.
 
-The backup hosts in use need to be defined in evobackup-client__hosts
-and the bkctld jail ssh port has to be defined in
-evobackup-client__ssh_port before running it.
+The backup hosts and the ports in use need to be defined in
+evobackup-client__hosts before running it.
 
-The default zzz_evobackup.sh configures a system backup, but the
+The default zzz_evobackup.sh configures a system only backup, but the
 template can be overriden to configure a full backup instead. If
 you change the variables in defaults/main.yml you can easily run
 this again and configure backups to a second set of hosts.
diff --git a/evobackup-client/defaults/main.yml b/evobackup-client/defaults/main.yml
index e11de5c9..c172d178 100644
--- a/evobackup-client/defaults/main.yml
+++ b/evobackup-client/defaults/main.yml
@@ -6,8 +6,8 @@ evobackup-client__mail: null
 evobackup-client__pid_path: "/var/run/evobackup.pid"
 evobackup-client__log_path: "/var/log/evobackup.log"
 evobackup-client__backup_path: "/home/backup"
-evobackup-client__ssh_port: null
 evobackup-client__hosts: null
 #  - name: "backups.example.org"
 #    ip: "xxx.xxx.xxx.xxx"
 #    fingerprint: "ecdsa-sha2-nistp256 ..."
+#    port: xxxx
diff --git a/evobackup-client/tasks/open_ssh_ports.yml b/evobackup-client/tasks/open_ssh_ports.yml
index 7979d233..22f3fb01 100644
--- a/evobackup-client/tasks/open_ssh_ports.yml
+++ b/evobackup-client/tasks/open_ssh_ports.yml
@@ -8,22 +8,12 @@
     - evobackup-client
     - evobackup-client-backup-firewall
 
-- name: backup ssh port
-  blockinfile:
-    dest: /etc/default/minifirewall
-    marker: "# evobackup ssh port"
-    block: "evobackup_port={{ evobackup-client__ssh_port }}"
-  when: evobackup-client__minifirewall.stat.exists
-  tags:
-    - evobackup-client
-    - evobackup-client-backup-firewall
-
 - name: Add backup SSH port in /etc/default/minifirewall
   blockinfile:
     dest: /etc/default/minifirewall
     marker: "# {{ item.name }}"
     block: |
-      /sbin/iptables -A INPUT -p tcp --sport {{ evobackup-client__ssh_port }} --dport 1024:65535 -s {{ item.ip }} -m state --state ESTABLISHED,RELATED -j ACCEPT
+      /sbin/iptables -A INPUT -p tcp --sport {{ item.port }} --dport 1024:65535 -s {{ item.ip }} -m state --state ESTABLISHED,RELATED -j ACCEPT
   with_items: "{{ evobackup-client__hosts }}"
   when: evobackup-client__minifirewall.stat.exists
   tags:
diff --git a/evobackup-client/tasks/verify_ssh.yml b/evobackup-client/tasks/verify_ssh.yml
index 99dd6cb4..df0318e2 100644
--- a/evobackup-client/tasks/verify_ssh.yml
+++ b/evobackup-client/tasks/verify_ssh.yml
@@ -3,8 +3,8 @@
 - name: Verify evolix backup servers
   known_hosts:
     path: /root/.ssh/known_hosts
-    name: "[{{ item.name }}]:{{ evobackup-client__ssh_port }}"
-    key: "[{{ item.name }}]:{{ evobackup-client__ssh_port }} {{ item.fingerprint }}"
+    name: "[{{ item.name }}]:{{ item.port }}"
+    key: "[{{ item.name }}]:{{ item.port }} {{ item.fingerprint }}"
   with_list: "{{ evobackup-client__hosts }}"
   tags:
     - evobackup-client
diff --git a/evobackup-client/templates/zzz_evobackup.default.sh.j2 b/evobackup-client/templates/zzz_evobackup.default.sh.j2
index 27346930..766c023d 100644
--- a/evobackup-client/templates/zzz_evobackup.default.sh.j2
+++ b/evobackup-client/templates/zzz_evobackup.default.sh.j2
@@ -21,7 +21,7 @@
 MAIL={{ evobackup-client__mail }}
 
 # list of hosts (hostname or IP) and SSH port for Rsync
-SERVERS="{% for host in evobackup-client__hosts %}{{ host.name }}:{{ evobackup-client__ssh_port }} {% endfor %}"
+SERVERS="{% for host in evobackup-client__hosts %}{{ host.name }}:{{ host.port }} {% endfor %}"
 
 # timeout (in seconds) for SSH connections
 SSH_CONNECT_TIMEOUT=30

From 71bf970811baf78a00d5ed53088ce9ac052c87fe Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Tue, 3 Sep 2019 11:14:48 -0400
Subject: [PATCH 05/17] Fix extra space in zzz_evobackup template

Not the best solution, a mix of map, format, join would of been
better, but I could not find out how to make it work with multiple
attributes.
---
 evobackup-client/templates/zzz_evobackup.default.sh.j2 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/evobackup-client/templates/zzz_evobackup.default.sh.j2 b/evobackup-client/templates/zzz_evobackup.default.sh.j2
index 766c023d..5471bec4 100644
--- a/evobackup-client/templates/zzz_evobackup.default.sh.j2
+++ b/evobackup-client/templates/zzz_evobackup.default.sh.j2
@@ -19,9 +19,9 @@
 
 # email adress for notifications
 MAIL={{ evobackup-client__mail }}
-
+{{ evobackup-client__hosts | map
 # list of hosts (hostname or IP) and SSH port for Rsync
-SERVERS="{% for host in evobackup-client__hosts %}{{ host.name }}:{{ host.port }} {% endfor %}"
+SERVERS="{% for host in evobackup-client__hosts %}{{ host.name }}:{{ host.port }}{% if loop.index != loop.length %} {% endif %}{% endfor %}"
 
 # timeout (in seconds) for SSH connections
 SSH_CONNECT_TIMEOUT=30

From f9b55c0c91a2853642612f4a0c9c61de81871169 Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Tue, 3 Sep 2019 12:13:05 -0400
Subject: [PATCH 06/17] Got rid of forgotten attempt at mapping hosts in
 evobackup-client

---
 evobackup-client/templates/zzz_evobackup.default.sh.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/evobackup-client/templates/zzz_evobackup.default.sh.j2 b/evobackup-client/templates/zzz_evobackup.default.sh.j2
index 5471bec4..951af7f3 100644
--- a/evobackup-client/templates/zzz_evobackup.default.sh.j2
+++ b/evobackup-client/templates/zzz_evobackup.default.sh.j2
@@ -19,7 +19,7 @@
 
 # email adress for notifications
 MAIL={{ evobackup-client__mail }}
-{{ evobackup-client__hosts | map
+
 # list of hosts (hostname or IP) and SSH port for Rsync
 SERVERS="{% for host in evobackup-client__hosts %}{{ host.name }}:{{ host.port }}{% if loop.index != loop.length %} {% endif %}{% endfor %}"
 

From d226ce594ae76dba402e0dafbaf9dbc27c620a94 Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Fri, 6 Sep 2019 17:49:40 -0400
Subject: [PATCH 07/17] Changed variables to all lower snake case for
 evobackup-client

Not just esthetic, for some reason ansible refused to run with '-'
somewhere and I just could not find where, changing variables and
tags to use '_' fixed it. It's more consistent anyway.
---
 evobackup-client/defaults/main.yml               | 16 ++++++++--------
 evobackup-client/tasks/main.yml                  | 16 ++++++++--------
 evobackup-client/tasks/open_ssh_ports.yml        | 14 +++++++-------
 evobackup-client/tasks/ssh_key.yml               | 16 ++++++++--------
 evobackup-client/tasks/upload_scripts.yml        | 12 ++++++------
 evobackup-client/tasks/verify_ssh.yml            |  6 +++---
 .../templates/zzz_evobackup.default.sh.j2        | 10 +++++-----
 7 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/evobackup-client/defaults/main.yml b/evobackup-client/defaults/main.yml
index c172d178..e48dc739 100644
--- a/evobackup-client/defaults/main.yml
+++ b/evobackup-client/defaults/main.yml
@@ -1,12 +1,12 @@
 ---
-evobackup-client__root_key_path: "/root/.ssh/evobackup_id"
-evobackup-client__cron_path: "/etc/cron.daily/zzz_evobackup"
-evobackup-client__cron_template_name: "zzz_evobackup"
-evobackup-client__mail: null
-evobackup-client__pid_path: "/var/run/evobackup.pid"
-evobackup-client__log_path: "/var/log/evobackup.log"
-evobackup-client__backup_path: "/home/backup"
-evobackup-client__hosts: null
+evobackup_client__root_key_path: "/root/.ssh/evobackup_id"
+evobackup_client__cron_path: "/etc/cron.daily/zzz_evobackup"
+evobackup_client__cron_template_name: "zzz_evobackup"
+evobackup_client__mail: null
+evobackup_client__pid_path: "/var/run/evobackup.pid"
+evobackup_client__log_path: "/var/log/evobackup.log"
+evobackup_client__backup_path: "/home/backup"
+evobackup_client__hosts: null
 #  - name: "backups.example.org"
 #    ip: "xxx.xxx.xxx.xxx"
 #    fingerprint: "ecdsa-sha2-nistp256 ..."
diff --git a/evobackup-client/tasks/main.yml b/evobackup-client/tasks/main.yml
index 9b25f2bf..8240a595 100644
--- a/evobackup-client/tasks/main.yml
+++ b/evobackup-client/tasks/main.yml
@@ -2,20 +2,20 @@
 
 - include: "ssh_key.yml"
   tags:
-    - evobackup-client
-    - evobackup-client-backup-ssh_key
+    - evobackup_client
+    - evobackup_client_backup_ssh_key
 
 - include: "upload_scripts.yml"
   tags:
-    - evobackup-client
-    - evobackup-client-backup-scripts
+    - evobackup_client
+    - evobackup_client_backup_scripts
 
 - include: "open_ssh_ports.yml"
   tags:
-    - evobackup-client
-    - evobackup-client-backup-firewall
+    - evobackup_client
+    - evobackup_client_backup_firewall
 
 - include: "verify_ssh.yml"
   tags:
-    - evobackup-client
-    - evobackup-client-backup-hosts
+    - evobackup_client
+    - evobackup_client_backup_hosts
diff --git a/evobackup-client/tasks/open_ssh_ports.yml b/evobackup-client/tasks/open_ssh_ports.yml
index 22f3fb01..14a7b304 100644
--- a/evobackup-client/tasks/open_ssh_ports.yml
+++ b/evobackup-client/tasks/open_ssh_ports.yml
@@ -3,10 +3,10 @@
 - name: Is there a Minifirewall ?
   stat:
     path: /etc/default/minifirewall
-  register: evobackup-client__minifirewall
+  register: evobackup_client__minifirewall
   tags:
-    - evobackup-client
-    - evobackup-client-backup-firewall
+    - evobackup_client
+    - evobackup_client_backup_firewall
 
 - name: Add backup SSH port in /etc/default/minifirewall
   blockinfile:
@@ -14,8 +14,8 @@
     marker: "# {{ item.name }}"
     block: |
       /sbin/iptables -A INPUT -p tcp --sport {{ item.port }} --dport 1024:65535 -s {{ item.ip }} -m state --state ESTABLISHED,RELATED -j ACCEPT
-  with_items: "{{ evobackup-client__hosts }}"
-  when: evobackup-client__minifirewall.stat.exists
+  with_items: "{{ evobackup_client__hosts }}"
+  when: evobackup_client__minifirewall.stat.exists
   tags:
-    - evobackup-client
-    - evobackup-client-backup-firewall
+    - evobackup_client
+    - evobackup_client_backup_firewall
diff --git a/evobackup-client/tasks/ssh_key.yml b/evobackup-client/tasks/ssh_key.yml
index 6322504f..2d629500 100644
--- a/evobackup-client/tasks/ssh_key.yml
+++ b/evobackup-client/tasks/ssh_key.yml
@@ -4,17 +4,17 @@
   user:
     name: root
     generate_ssh_key: true
-    ssh_key_file: "{{ evobackup-client__root_key_path }}"
+    ssh_key_file: "{{ evobackup_client__root_key_path }}"
     ssh_key_type: rsa
-  register: evobackup-client__root_key
+  register: evobackup_client__root_key
   tags:
-    - evobackup-client
-    - evobackup-client-backup-ssh_key
+    - evobackup_client
+    - evobackup_client_backup_ssh_key
 
 - name: Print SSH key
   debug:
-    var: evobackup-client__root_key.ssh_public_key
-  when: evobackup-client__root_key.ssh_public_key is defined
+    var: evobackup_client__root_key.ssh_public_key
+  when: evobackup_client__root_key.ssh_public_key is defined
   tags:
-    - evobackup-client
-    - evobackup-client-backup-ssh_key
+    - evobackup_client
+    - evobackup_client_backup_ssh_key
diff --git a/evobackup-client/tasks/upload_scripts.yml b/evobackup-client/tasks/upload_scripts.yml
index 2bb94d6e..8d698519 100644
--- a/evobackup-client/tasks/upload_scripts.yml
+++ b/evobackup-client/tasks/upload_scripts.yml
@@ -3,14 +3,14 @@
 - name: Upload evobackup script
   template:
     src: "{{ item }}"
-    dest: "{{ evobackup-client__cron_path }}"
+    dest: "{{ evobackup_client__cron_path }}"
     force: true
     mode: 0755
   with_first_found:
-    - "templates/evobackup-client/{{ evobackup-client__cron_template_name }}.{{ inventory_hostname }}.sh.j2"
-    - "templates/evobackup-client/{{ evobackup-client__cron_template_name }}.{{ host_group }}.sh.j2"
-    - "templates/evobackup-client/{{ evobackup-client__cron_template_name }}.sh.j2"
+    - "templates/evobackup-client/{{ evobackup_client__cron_template_name }}.{{ inventory_hostname }}.sh.j2"
+    - "templates/evobackup-client/{{ evobackup_client__cron_template_name }}.{{ host_group }}.sh.j2"
+    - "templates/evobackup-client/{{ evobackup_client__cron_template_name }}.sh.j2"
     - "zzz_evobackup.default.sh.j2"
   tags:
-    - evobackup-client
-    - evobackup-client-backup-scripts
+    - evobackup_client
+    - evobackup_client_backup_scripts
diff --git a/evobackup-client/tasks/verify_ssh.yml b/evobackup-client/tasks/verify_ssh.yml
index df0318e2..4e968197 100644
--- a/evobackup-client/tasks/verify_ssh.yml
+++ b/evobackup-client/tasks/verify_ssh.yml
@@ -5,7 +5,7 @@
     path: /root/.ssh/known_hosts
     name: "[{{ item.name }}]:{{ item.port }}"
     key: "[{{ item.name }}]:{{ item.port }} {{ item.fingerprint }}"
-  with_list: "{{ evobackup-client__hosts }}"
+  with_list: "{{ evobackup_client__hosts }}"
   tags:
-    - evobackup-client
-    - evobackup-client-backup-hosts
+    - evobackup_client
+    - evobackup_client_backup_hosts
diff --git a/evobackup-client/templates/zzz_evobackup.default.sh.j2 b/evobackup-client/templates/zzz_evobackup.default.sh.j2
index 951af7f3..c2aa59de 100644
--- a/evobackup-client/templates/zzz_evobackup.default.sh.j2
+++ b/evobackup-client/templates/zzz_evobackup.default.sh.j2
@@ -18,23 +18,23 @@
 ##### Configuration ###################################################
 
 # email adress for notifications
-MAIL={{ evobackup-client__mail }}
+MAIL={{ evobackup_client__mail }}
 
 # list of hosts (hostname or IP) and SSH port for Rsync
-SERVERS="{% for host in evobackup-client__hosts %}{{ host.name }}:{{ host.port }}{% if loop.index != loop.length %} {% endif %}{% endfor %}"
+SERVERS="{% for host in evobackup_client__hosts %}{{ host.name }}:{{ host.port }}{% if loop.index != loop.length %} {% endif %}{% endfor %}"
 
 # timeout (in seconds) for SSH connections
 SSH_CONNECT_TIMEOUT=30
 
 ## We use /home/backup : feel free to use your own dir
-LOCAL_BACKUP_DIR="{{ evobackup-client__backup_path }}"
+LOCAL_BACKUP_DIR="{{ evobackup_client__backup_path }}"
 
 # You can set "linux" or "bsd" manually or let it choose automatically
 SYSTEM=$(uname | tr '[:upper:]' '[:lower:]')
 
 # Change these 2 variables if you have more than one backup cron
-PIDFILE="{{ evobackup-client__pid_path }}"
-LOGFILE="{{ evobackup-client__log_path }}"
+PIDFILE="{{ evobackup_client__pid_path }}"
+LOGFILE="{{ evobackup_client__log_path }}"
 
 ## Enable/Disable tasks
 LOCAL_TASKS=${LOCAL_TASKS:-1}

From 11e006201af2f27ddaff769d2aece0e8dac3b43d Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Fri, 6 Sep 2019 17:52:14 -0400
Subject: [PATCH 08/17] Revert to default zzz_evobackup from git

The suggested default is better, following peer review. Though some
changes will probably need to be made once we decide how to handle
system only backups.
---
 evobackup-client/templates/zzz_evobackup.default.sh.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/evobackup-client/templates/zzz_evobackup.default.sh.j2 b/evobackup-client/templates/zzz_evobackup.default.sh.j2
index c2aa59de..b120d211 100644
--- a/evobackup-client/templates/zzz_evobackup.default.sh.j2
+++ b/evobackup-client/templates/zzz_evobackup.default.sh.j2
@@ -341,7 +341,7 @@ if [ "${SYNC_TASKS}" = "1" ]; then
             /etc                                           \
             /root                                          \
             /var                                           \
-            /home/backup                                          \
+            /home                                          \
             /srv                                           \
         -e "${RSH_COMMAND}"                                \
         "root@${SSH_SERVER}:/var/backup/"                  \

From b36202f8d1802927e6737865ce278ebf9aa31f01 Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Fri, 6 Sep 2019 17:53:48 -0400
Subject: [PATCH 09/17] minifirewall was not restarted by evobackup-client
 after mods.

---
 evobackup-client/tasks/open_ssh_ports.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/evobackup-client/tasks/open_ssh_ports.yml b/evobackup-client/tasks/open_ssh_ports.yml
index 14a7b304..4ab58c39 100644
--- a/evobackup-client/tasks/open_ssh_ports.yml
+++ b/evobackup-client/tasks/open_ssh_ports.yml
@@ -15,6 +15,7 @@
     block: |
       /sbin/iptables -A INPUT -p tcp --sport {{ item.port }} --dport 1024:65535 -s {{ item.ip }} -m state --state ESTABLISHED,RELATED -j ACCEPT
   with_items: "{{ evobackup_client__hosts }}"
+  notifiy: restart minifirewall
   when: evobackup_client__minifirewall.stat.exists
   tags:
     - evobackup_client

From 87202fa264f1dccf662b09e4b75311e3aa8e00dc Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Mon, 23 Sep 2019 15:26:30 -0400
Subject: [PATCH 10/17] fixed typo in evobackup-cilent

Not sure how this slipped in
---
 evobackup-client/tasks/open_ssh_ports.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/evobackup-client/tasks/open_ssh_ports.yml b/evobackup-client/tasks/open_ssh_ports.yml
index 4ab58c39..0f8b0cda 100644
--- a/evobackup-client/tasks/open_ssh_ports.yml
+++ b/evobackup-client/tasks/open_ssh_ports.yml
@@ -15,7 +15,7 @@
     block: |
       /sbin/iptables -A INPUT -p tcp --sport {{ item.port }} --dport 1024:65535 -s {{ item.ip }} -m state --state ESTABLISHED,RELATED -j ACCEPT
   with_items: "{{ evobackup_client__hosts }}"
-  notifiy: restart minifirewall
+  notify: restart minifirewall
   when: evobackup_client__minifirewall.stat.exists
   tags:
     - evobackup_client

From 4851af7432e91f32f214a07b650df26a3610ef1c Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Thu, 3 Oct 2019 10:44:21 -0400
Subject: [PATCH 11/17] Fix minifirewall restart handler

---
 evobackup-client/handlers/main.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/evobackup-client/handlers/main.yml b/evobackup-client/handlers/main.yml
index 5a0c6bfd..4ddd1f53 100644
--- a/evobackup-client/handlers/main.yml
+++ b/evobackup-client/handlers/main.yml
@@ -1,5 +1,6 @@
 ---
 - name: restart minifirewall
-  service:
-    name: minifirewall
-    state: restarted
+  command: /etc/init.d/minifirewall restart
+  register: minifirewall_init_restart
+  failed_when: "'starting IPTables rules is now finish : OK' not in minifirewall_init_restart.stdout"
+  changed_when: "'starting IPTables rules is now finish : OK' in minifirewall_init_restart.stdout"

From daad12fdeb892c3025b0d075af234eedd1fc0ed8 Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Mon, 7 Oct 2019 12:28:25 -0400
Subject: [PATCH 12/17] Handle backup jail creation

Does not handle sync step yet
---
 evobackup-client/handlers/main.yml        |  4 ++++
 evobackup-client/tasks/create_jail.yml    | 29 +++++++++++++++++++++++
 evobackup-client/tasks/main.yml           |  5 ++++
 evobackup-client/tasks/open_ssh_ports.yml |  2 +-
 evobackup-client/tasks/ssh_key.yml        | 11 +++++++++
 5 files changed, 50 insertions(+), 1 deletion(-)
 create mode 100644 evobackup-client/tasks/create_jail.yml

diff --git a/evobackup-client/handlers/main.yml b/evobackup-client/handlers/main.yml
index 4ddd1f53..9d0e022f 100644
--- a/evobackup-client/handlers/main.yml
+++ b/evobackup-client/handlers/main.yml
@@ -4,3 +4,7 @@
   register: minifirewall_init_restart
   failed_when: "'starting IPTables rules is now finish : OK' not in minifirewall_init_restart.stdout"
   changed_when: "'starting IPTables rules is now finish : OK' in minifirewall_init_restart.stdout"
+
+- name: 'created new jail'
+  command: "bkctld start {{ evolinux_hostname }}"
+  delegate_to: "{{ evobackup_client__hosts[0].ip }}"
diff --git a/evobackup-client/tasks/create_jail.yml b/evobackup-client/tasks/create_jail.yml
new file mode 100644
index 00000000..df5902e4
--- /dev/null
+++ b/evobackup-client/tasks/create_jail.yml
@@ -0,0 +1,29 @@
+---
+
+- name: 'create jail'
+  command: "bkctld init {{ evolinux_hostname }}"
+  args:
+    creates: "/backup/jails/{{ evolinux_hostname }}/"
+  become: true
+  delegate_to: "{{ evobackup_client__hosts[0].ip }}"
+  notify: 'created new jail'
+
+- name: 'add ssh key to jail'
+  command: "bkctld key {{ evolinux_hostname }} /root/{{ evolinux_hostname }}.pub"
+  become: true
+  delegate_to: "{{ evobackup_client__hosts[0].ip }}"
+
+- name: 'add ip to jail'
+  command: "bkctld ip {{ evolinux_hostname }} {{ ansible_host }}"
+  become: true
+  delegate_to: "{{ evobackup_client__hosts[0].ip }}"
+
+- name: 'get jail port'
+  command: "bkctld port {{ evolinux_hostname }}"
+  become: true
+  register: bkctld_port
+  delegate_to: "{{ evobackup_client__hosts[0].ip }}"
+
+- name: 'register jail port'
+  set_fact:
+    evobackup_ssh_port={{ bkctld_port.stdout }}
diff --git a/evobackup-client/tasks/main.yml b/evobackup-client/tasks/main.yml
index 8240a595..69bcd16e 100644
--- a/evobackup-client/tasks/main.yml
+++ b/evobackup-client/tasks/main.yml
@@ -5,6 +5,11 @@
     - evobackup_client
     - evobackup_client_backup_ssh_key
 
+- include: "create_jail.yml"
+  tags:
+    - evobackup_client
+    - evobackup_client_jail
+
 - include: "upload_scripts.yml"
   tags:
     - evobackup_client
diff --git a/evobackup-client/tasks/open_ssh_ports.yml b/evobackup-client/tasks/open_ssh_ports.yml
index 0f8b0cda..be96c161 100644
--- a/evobackup-client/tasks/open_ssh_ports.yml
+++ b/evobackup-client/tasks/open_ssh_ports.yml
@@ -11,7 +11,7 @@
 - name: Add backup SSH port in /etc/default/minifirewall
   blockinfile:
     dest: /etc/default/minifirewall
-    marker: "# {{ item.name }}"
+    marker: "# {mark} {{ item.name }}"
     block: |
       /sbin/iptables -A INPUT -p tcp --sport {{ item.port }} --dport 1024:65535 -s {{ item.ip }} -m state --state ESTABLISHED,RELATED -j ACCEPT
   with_items: "{{ evobackup_client__hosts }}"
diff --git a/evobackup-client/tasks/ssh_key.yml b/evobackup-client/tasks/ssh_key.yml
index 2d629500..6327dd74 100644
--- a/evobackup-client/tasks/ssh_key.yml
+++ b/evobackup-client/tasks/ssh_key.yml
@@ -18,3 +18,14 @@
   tags:
     - evobackup_client
     - evobackup_client_backup_ssh_key
+
+- name: 'copy ssh public key to backup server'
+  copy:
+    content: "{{ evobackup_client__root_key.ssh_public_key }}"
+    dest: "/root/{{ evolinux_hostname }}.pub"
+  become: true
+  delegate_to: "{{ evobackup_client__hosts[0].ip }}"
+  tags:
+    - evobackup_client
+    - evobackup_client_backup_ssh_key
+    - evobackup_client_jail

From d4742b411d9b3ed09ced1289d788796cb4986975 Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Mon, 7 Oct 2019 13:53:10 -0400
Subject: [PATCH 13/17] Added evobackup sync when more than one host

---
 evobackup-client/handlers/main.yml     |  5 +++++
 evobackup-client/tasks/create_jail.yml | 21 ++++++++++++++++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/evobackup-client/handlers/main.yml b/evobackup-client/handlers/main.yml
index 9d0e022f..c3ee4ec7 100644
--- a/evobackup-client/handlers/main.yml
+++ b/evobackup-client/handlers/main.yml
@@ -8,3 +8,8 @@
 - name: 'created new jail'
   command: "bkctld start {{ evolinux_hostname }}"
   delegate_to: "{{ evobackup_client__hosts[0].ip }}"
+
+- name: 'jail updated'
+  command: "bkctld start {{ evolinux_hostname }}"
+  delegate_to: "{{ evobackup_client__hosts[0].ip }}"
+  when: evobackup_client__hosts|length > 1
diff --git a/evobackup-client/tasks/create_jail.yml b/evobackup-client/tasks/create_jail.yml
index df5902e4..2c78a7c1 100644
--- a/evobackup-client/tasks/create_jail.yml
+++ b/evobackup-client/tasks/create_jail.yml
@@ -6,24 +6,43 @@
     creates: "/backup/jails/{{ evolinux_hostname }}/"
   become: true
   delegate_to: "{{ evobackup_client__hosts[0].ip }}"
-  notify: 'created new jail'
+  notify:
+    - 'created new jail'
+    - 'jail updated'
+  tags:
+    - evobackup_client
+    - evobackup_client_jail
 
 - name: 'add ssh key to jail'
   command: "bkctld key {{ evolinux_hostname }} /root/{{ evolinux_hostname }}.pub"
   become: true
   delegate_to: "{{ evobackup_client__hosts[0].ip }}"
+  notify:  'jail updated'
+  tags:
+    - evobackup_client
+    - evobackup_client_jail
 
 - name: 'add ip to jail'
   command: "bkctld ip {{ evolinux_hostname }} {{ ansible_host }}"
   become: true
   delegate_to: "{{ evobackup_client__hosts[0].ip }}"
+  notify:  'jail updated'
+  tags:
+    - evobackup_client
+    - evobackup_client_jail
 
 - name: 'get jail port'
   command: "bkctld port {{ evolinux_hostname }}"
   become: true
   register: bkctld_port
   delegate_to: "{{ evobackup_client__hosts[0].ip }}"
+  tags:
+    - evobackup_client
+    - evobackup_client_jail
 
 - name: 'register jail port'
   set_fact:
     evobackup_ssh_port={{ bkctld_port.stdout }}
+  tags:
+    - evobackup_client
+    - evobackup_client_jail

From 5a20292ec9cf5bf2f9dab8b3da0b5a1b0130416f Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Mon, 7 Oct 2019 13:53:55 -0400
Subject: [PATCH 14/17] rename create_jail.yml to jail.yml in evobackup-client

---
 evobackup-client/tasks/{create_jail.yml => jail.yml} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename evobackup-client/tasks/{create_jail.yml => jail.yml} (100%)

diff --git a/evobackup-client/tasks/create_jail.yml b/evobackup-client/tasks/jail.yml
similarity index 100%
rename from evobackup-client/tasks/create_jail.yml
rename to evobackup-client/tasks/jail.yml

From 5178e99e7c19050b018f34ee6af26d6c7135c3f2 Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand+git@evolix.ca>
Date: Thu, 10 Oct 2019 15:40:00 -0400
Subject: [PATCH 15/17] typo in evobackup jail file task name

---
 evobackup-client/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/evobackup-client/tasks/main.yml b/evobackup-client/tasks/main.yml
index 69bcd16e..a2dd4405 100644
--- a/evobackup-client/tasks/main.yml
+++ b/evobackup-client/tasks/main.yml
@@ -5,7 +5,7 @@
     - evobackup_client
     - evobackup_client_backup_ssh_key
 
-- include: "create_jail.yml"
+- include: "jail.yml"
   tags:
     - evobackup_client
     - evobackup_client_jail

From 0b4095d8dd4e8a6e4d7d275c42fd7f5a3fc5cb31 Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand@evolix.ca>
Date: Thu, 19 Dec 2019 10:30:58 -0500
Subject: [PATCH 16/17] bkctld restart works better than bkctld start

a simple start will fail if the jail is already started, there is potential for a race condition.
---
 evobackup-client/handlers/main.yml | 4 ++--
 evobackup-client/tasks/jail.yml    | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/evobackup-client/handlers/main.yml b/evobackup-client/handlers/main.yml
index c3ee4ec7..54f7e44e 100644
--- a/evobackup-client/handlers/main.yml
+++ b/evobackup-client/handlers/main.yml
@@ -6,10 +6,10 @@
   changed_when: "'starting IPTables rules is now finish : OK' in minifirewall_init_restart.stdout"
 
 - name: 'created new jail'
-  command: "bkctld start {{ evolinux_hostname }}"
+  command: "bkctld restart {{ evolinux_hostname }}"
   delegate_to: "{{ evobackup_client__hosts[0].ip }}"
 
 - name: 'jail updated'
-  command: "bkctld start {{ evolinux_hostname }}"
+  command: "bkctld restart {{ evolinux_hostname }}"
   delegate_to: "{{ evobackup_client__hosts[0].ip }}"
   when: evobackup_client__hosts|length > 1
diff --git a/evobackup-client/tasks/jail.yml b/evobackup-client/tasks/jail.yml
index 2c78a7c1..53609a63 100644
--- a/evobackup-client/tasks/jail.yml
+++ b/evobackup-client/tasks/jail.yml
@@ -8,7 +8,6 @@
   delegate_to: "{{ evobackup_client__hosts[0].ip }}"
   notify:
     - 'created new jail'
-    - 'jail updated'
   tags:
     - evobackup_client
     - evobackup_client_jail

From af61b7f97d45505a45c4b854df2183839c64e94f Mon Sep 17 00:00:00 2001
From: Patrick Marchand <pmarchand@evolix.ca>
Date: Thu, 16 Jan 2020 12:18:51 +0100
Subject: [PATCH 17/17] Temp fix for regression introduced by bkctld 2.x

The erroneous error codes returned by bkctld will make playbooks
fail even though the commands run succesfully.
See https://gitea.evolix.org/evolix/evobackup/issues/31
---
 evobackup-client/tasks/jail.yml | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/evobackup-client/tasks/jail.yml b/evobackup-client/tasks/jail.yml
index 53609a63..82607705 100644
--- a/evobackup-client/tasks/jail.yml
+++ b/evobackup-client/tasks/jail.yml
@@ -12,11 +12,24 @@
     - evobackup_client
     - evobackup_client_jail
 
-- name: 'add ssh key to jail'
-  command: "bkctld key {{ evolinux_hostname }} /root/{{ evolinux_hostname }}.pub"
+- name: 'create jail'
+  command: "bkctld init {{ evolinux_hostname }}"
+  args:
+    creates: "/backup/jails/{{ evolinux_hostname }}/"
+  become: true
+  delegate_to: "{{ evobackup_client__hosts[0].ip }}"
+  notify:
+    - 'created new jail'
+  tags:
+    - evobackup_client
+    - evobackup_client_jail
+
+# temp fix for bkctld 2.x because the ip and key command return 1
+# if the jail is not started, see https://gitea.evolix.org/evolix/evobackup/issues/31
+- name: 'start jail'
+  command: "bkctld restart {{ evolinux_hostname }}"
   become: true
   delegate_to: "{{ evobackup_client__hosts[0].ip }}"
-  notify:  'jail updated'
   tags:
     - evobackup_client
     - evobackup_client_jail