From 78326e43e8615a344218563949e071c9be3454e0 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 1 Mar 2024 09:09:52 +0100 Subject: [PATCH 01/40] wording in the CHANGELOG --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8362c5d2..a49c669c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). This project does not follow semantic versioning. The **major** part of the version is the year -The **minor** part changes is the month -The **patch** part changes is incremented if multiple releases happen the same month +The **minor** part is the month +The **patch** part is incremented if multiple releases happen the same month ## [Unreleased] From 8b68f039105ef4ec996b3793abc114278b16a398 Mon Sep 17 00:00:00 2001 From: Alexis Ben Miloud--Josselin Date: Mon, 4 Mar 2024 15:00:01 +0100 Subject: [PATCH 02/40] certbot: Fix HAProxy renewal hook --- CHANGELOG.md | 2 ++ certbot/files/hooks/deploy/haproxy.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a49c669c..a8537706 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ The **patch** part is incremented if multiple releases happen the same month ### Fixed +* certbot: Fix HAProxy renewal hook + ### Removed ### Security diff --git a/certbot/files/hooks/deploy/haproxy.sh b/certbot/files/hooks/deploy/haproxy.sh index c08fafc2..36a09262 100644 --- a/certbot/files/hooks/deploy/haproxy.sh +++ b/certbot/files/hooks/deploy/haproxy.sh @@ -30,7 +30,7 @@ concat_files() { } cert_and_key_mismatch() { haproxy_cert_md5=$(openssl x509 -noout -pubkey -in "${haproxy_cert_file}" | openssl md5) - haproxy_key_md5=$(openssl pkey -noout -pubout -in "${haproxy_cert_file}" | openssl md5) + haproxy_key_md5=$(openssl pkey -pubout -in "${haproxy_cert_file}" | openssl md5) test "${haproxy_cert_md5}" != "${haproxy_key_md5}" } From 501f5e7577bfd193f698f97e4bbffb83c101f1a2 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 5 Mar 2024 16:54:10 +0100 Subject: [PATCH 03/40] autosysadmin-agent: upstream release 24.03.1 --- CHANGELOG.md | 2 ++ autosysadmin-agent/files/upstream/lib/common.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8537706..4c623801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ The **patch** part is incremented if multiple releases happen the same month ### Changed +* autosysadmin-agent: upstream release 24.03.1 + ### Fixed * certbot: Fix HAProxy renewal hook diff --git a/autosysadmin-agent/files/upstream/lib/common.sh b/autosysadmin-agent/files/upstream/lib/common.sh index cc3c53e6..9a7c7e23 100755 --- a/autosysadmin-agent/files/upstream/lib/common.sh +++ b/autosysadmin-agent/files/upstream/lib/common.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION="24.03" +VERSION="24.03.1" # Common functions for "repair" and "restart" scripts @@ -83,7 +83,7 @@ initialize() { LOCK_WAIT="0" # Default values for email headers - EMAIL_FROM="equipe+autosysadmin@evolix.fr" + EMAIL_FROM="equipe+autosysadmin@evolix.net" EMAIL_INTERNAL="autosysadmin@evolix.fr" LOCK_FILE="${RUN_DIR}/${LOCK_NAME}.lock" From 2fe0d252774bb12b6e1f34a1fc8f3ec06a11de5e Mon Sep 17 00:00:00 2001 From: Tom David--Broglio Date: Thu, 7 Mar 2024 15:47:39 +0100 Subject: [PATCH 04/40] correction tasks keepalived --- CHANGELOG.md | 1 + keepalived/tasks/main.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c623801..19caa254 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The **patch** part is incremented if multiple releases happen the same month ### Fixed * certbot: Fix HAProxy renewal hook +* keepalived: Fix tasks that use file instead of copy ### Removed diff --git a/keepalived/tasks/main.yml b/keepalived/tasks/main.yml index 30f9557a..3f436d0e 100644 --- a/keepalived/tasks/main.yml +++ b/keepalived/tasks/main.yml @@ -8,7 +8,7 @@ - keepalived - name: Add notify.sh script for NRPE check - ansible.builtin.file: + ansible.builtin.copy: src: notify.sh dest: /etc/keepalived/notify.sh mode: "0755" @@ -21,7 +21,7 @@ - nrpe - name: check_keepalived is installed - ansible.builtin.file: + ansible.builtin.copy: src: check_keepalived dest: /usr/local/lib/nagios/plugins/check_keepalived mode: "0755" From ccff3b21056129caa1b126e09fe3d2d81f227943 Mon Sep 17 00:00:00 2001 From: Gregory Colpart Date: Sun, 10 Mar 2024 12:38:13 +0100 Subject: [PATCH 05/40] =?UTF-8?q?Suppression=20de=20fichiers=20d=C3=A9sorm?= =?UTF-8?q?ais=20inutilis=C3=A9s=20(ils=20sont=20vides)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- postfix/files/spam.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postfix/files/spam.sh b/postfix/files/spam.sh index 29dc0b40..b8685f85 100644 --- a/postfix/files/spam.sh +++ b/postfix/files/spam.sh @@ -40,7 +40,7 @@ function cleanup { } # Postfix -postfix_dbs="client.access sender.access recipient.access header_kill sa-blacklist.access spamd.cidr" +postfix_dbs="client.access sender.access recipient.access header_kill" for db in ${postfix_dbs}; do if is_new "${db}"; then download "${db}" From aca146adbcd6e9decb657d71edfa0b833e06816a Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Mon, 11 Mar 2024 10:54:36 +0100 Subject: [PATCH 06/40] memcached: Fix conditions not properly writen (installation was always in multi-instance mode) --- CHANGELOG.md | 1 + memcached/tasks/main.yml | 4 ++-- memcached/tasks/munin.yml | 2 +- memcached/tasks/nrpe.yml | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19caa254..dccbbc99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The **patch** part is incremented if multiple releases happen the same month * certbot: Fix HAProxy renewal hook * keepalived: Fix tasks that use file instead of copy +* memcached: Fix conditions not properly writen (installation was always in multi-instance mode) ### Removed diff --git a/memcached/tasks/main.yml b/memcached/tasks/main.yml index 96060d4a..05455c0b 100644 --- a/memcached/tasks/main.yml +++ b/memcached/tasks/main.yml @@ -6,10 +6,10 @@ - memcached - ansible.builtin.include: instance-default.yml - when: memcached_instance_name is undefined + when: memcached_instance_name | length == 0 - ansible.builtin.include: instance-multi.yml - when: memcached_instance_name is defined + when: memcached_instance_name | length > 0 - ansible.builtin.include: munin.yml diff --git a/memcached/tasks/munin.yml b/memcached/tasks/munin.yml index b25b9275..f4b262a9 100644 --- a/memcached/tasks/munin.yml +++ b/memcached/tasks/munin.yml @@ -2,7 +2,7 @@ - name: Choose packages (Oracle) ansible.builtin.set_fact: multi: "multi_" - when: memcached_instance_name is defined + when: memcached_instance_name | length > 0 - name: is Munin present ? ansible.builtin.stat: diff --git a/memcached/tasks/nrpe.yml b/memcached/tasks/nrpe.yml index aba43da6..70c69c00 100644 --- a/memcached/tasks/nrpe.yml +++ b/memcached/tasks/nrpe.yml @@ -36,7 +36,7 @@ regexp: '^command\[check_memcached\]=' line: 'command[check_memcached]=/usr/local/lib/nagios/plugins/check_memcached.pl -H 127.0.0.1 -p {{ memcached_port }}' notify: restart nagios-nrpe-server - when: memcached_instance_name is undefined + when: memcached_instance_name | length == 0 - name: Add NRPE check (multi instance) ansible.builtin.lineinfile: @@ -44,6 +44,6 @@ regexp: '^command\[check_memcached\]=' line: 'command[check_memcached]=/usr/local/lib/nagios/plugins/check_memcached_instances' notify: restart nagios-nrpe-server - when: memcached_instance_name is defined + when: memcached_instance_name | length > 0 when: nrpe_evolix_config.stat.exists From bc9609ce48f6572b5989ca2d781f3c87347d6894 Mon Sep 17 00:00:00 2001 From: Alexis Ben Miloud--Josselin Date: Thu, 14 Mar 2024 10:16:10 +0100 Subject: [PATCH 07/40] nextcloud: use latest version by default --- CHANGELOG.md | 1 + webapps/nextcloud/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dccbbc99..908d890d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ The **patch** part is incremented if multiple releases happen the same month * lxc-php, php: Update sury PGP key * openvpn: earlier alert for CA expiration * redis: create sysfs config file if missing +* nextcloud: use latest version by default ### Removed diff --git a/webapps/nextcloud/defaults/main.yml b/webapps/nextcloud/defaults/main.yml index d9366933..2c70832c 100644 --- a/webapps/nextcloud/defaults/main.yml +++ b/webapps/nextcloud/defaults/main.yml @@ -1,5 +1,5 @@ --- -nextcloud_version: "latest-24" +nextcloud_version: "latest" nextcloud_archive_name: "{{ nextcloud_version }}.tar.bz2" nextcloud_releases_baseurl: "https://download.nextcloud.com/server/releases/" From eda69725d551c349b351a75244bfd6b03816c158 Mon Sep 17 00:00:00 2001 From: Eric Morino Date: Fri, 15 Mar 2024 09:19:55 +0100 Subject: [PATCH 08/40] proftpd: add whitelist ip in virtualhost sftp --- CHANGELOG.md | 1 + proftpd/tasks/accounts.yml | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 908d890d..f305b5a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ The **patch** part is incremented if multiple releases happen the same month * autosysadmin-restart_nrpe: add role * certbot: Renewal hook for NRPE * kvm-host: add minifirewall rules if DRBD interface is configured +* proftpd: add whitelist ip ### Changed diff --git a/proftpd/tasks/accounts.yml b/proftpd/tasks/accounts.yml index fc97b55b..11b2f60d 100644 --- a/proftpd/tasks/accounts.yml +++ b/proftpd/tasks/accounts.yml @@ -61,6 +61,27 @@ tags: - proftpd +- name: Whitelist ip for users (SFTP) + ansible.builtin.blockinfile: + dest: /etc/proftpd/conf.d/sftp.conf + marker: "# {mark} ANSIBLE MANAGED BLOCK - Whitelist ip for users" + block: | + {% for user in proftpd_accounts_final %} + {% if user.group is defined %} + + + {% for ip in proftpd_sftp_ips_whitelist[user.group] %} + Allow from {{ ip }} + {% endfor %} + DenyAll + + + {% endif %} + {% endfor %} + insertbefore: "" + notify: restart proftpd + when: proftpd_sftp_enable_user_whitelist | bool + - name: Allow keys for SFTP account ansible.builtin.template: dest: "/etc/proftpd/sftp.authorized_keys/{{ _proftpd_account.name }}" From ff19df24445bcf3baef856b75e9a8a55d40c69b3 Mon Sep 17 00:00:00 2001 From: Eric Morino Date: Mon, 18 Mar 2024 09:24:13 +0100 Subject: [PATCH 09/40] Add proftpd_sftp_enable_user_whitelist : False by default --- proftpd/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/proftpd/defaults/main.yml b/proftpd/defaults/main.yml index 0bcaa40f..9cc66b91 100644 --- a/proftpd/defaults/main.yml +++ b/proftpd/defaults/main.yml @@ -16,3 +16,4 @@ proftpd_sftp_use_publickeys: True proftpd_sftp_port: 22222 proftpd_accounts: [] proftpd_accounts_final: [] +proftpd_sftp_enable_user_whitelist : False From 8e3724d5e753a7702a2ace134fc568314f4ecda9 Mon Sep 17 00:00:00 2001 From: Eric Morino Date: Mon, 18 Mar 2024 09:50:43 +0100 Subject: [PATCH 10/40] Add documentation for add account proftpd with ips whitelist --- proftpd/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/proftpd/README.md b/proftpd/README.md index 6e96e05a..233462a3 100644 --- a/proftpd/README.md +++ b/proftpd/README.md @@ -43,3 +43,19 @@ For generate the sha512 version of yours password : ~~~ printf "test" | mkpasswd --stdin --method=sha-512 ~~~ + +## Add whitelist ip for accounts + +If you want add an filtering by ip for accounts, you have to enabled variable `proftpd_sftp_enable_user_whitelist` and add variable `proftpd_sftp_ips_whitelist` and a group by accounts. + +Example : + +~~~ +proftpd_sftp_enable_user_whitelist : True + +proftpd_sftp_ips_whitelist: + foo: ['127.0.0.1', '192.168.0.1'] + +proftpd_accounts: +- { name: 'ftp3', home: '/home/ftp3/', uid: 116, gid: 65534, group: 'foo', password: '$6$/Yy0b0No3GWh$3ZY1GZFI25eyQDBrANyHw.NFPqPqdg6sCi89nM/aNitmESZ2jGfROveS5xowy.WjX9tMC7.KPoabKPyxOpBJY0' } +~~~ From ae2e447bc402983e5b32309c8643bafee2788f60 Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Mon, 18 Mar 2024 15:30:23 +0100 Subject: [PATCH 11/40] evolinux-base: Add new variable to disable global customisation of bash config --- CHANGELOG.md | 1 + evolinux-base/defaults/main.yml | 3 +++ evolinux-base/tasks/main.yml | 1 + 3 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f305b5a4..0c7d2f44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The **patch** part is incremented if multiple releases happen the same month ### Changed * autosysadmin-agent: upstream release 24.03.1 +* evolinux-base: Add new variable to disable global customisation of bash config ### Fixed diff --git a/evolinux-base/defaults/main.yml b/evolinux-base/defaults/main.yml index 97f5540e..a706e319 100644 --- a/evolinux-base/defaults/main.yml +++ b/evolinux-base/defaults/main.yml @@ -142,6 +142,9 @@ evolinux_ssh_group: "evolinux-ssh" # # evolinux_users_include: True +# bash +evolinux_bash_config_include: true + # root evolinux_root_include: True diff --git a/evolinux-base/tasks/main.yml b/evolinux-base/tasks/main.yml index 456207df..931733c9 100644 --- a/evolinux-base/tasks/main.yml +++ b/evolinux-base/tasks/main.yml @@ -76,6 +76,7 @@ - name: Bash configuration ansible.builtin.import_tasks: bash.yml + when: evolinux_bash_config_include | bool - name: Root user configuration ansible.builtin.import_tasks: root.yml From d758afdd4bf431aa8bc034103c5810dec125d83d Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 19 Mar 2024 08:18:50 +0100 Subject: [PATCH 12/40] autosysadmin-agent: upstream release 24.03.2 --- CHANGELOG.md | 2 +- autosysadmin-agent/files/upstream/lib/common.sh | 2 +- autosysadmin-agent/files/upstream/lib/repair.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c7d2f44..97cac326 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ The **patch** part is incremented if multiple releases happen the same month ### Changed -* autosysadmin-agent: upstream release 24.03.1 +* autosysadmin-agent: upstream release 24.03.2 * evolinux-base: Add new variable to disable global customisation of bash config ### Fixed diff --git a/autosysadmin-agent/files/upstream/lib/common.sh b/autosysadmin-agent/files/upstream/lib/common.sh index 9a7c7e23..3ce25909 100755 --- a/autosysadmin-agent/files/upstream/lib/common.sh +++ b/autosysadmin-agent/files/upstream/lib/common.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION="24.03.1" +VERSION="24.03.2" # Common functions for "repair" and "restart" scripts diff --git a/autosysadmin-agent/files/upstream/lib/repair.sh b/autosysadmin-agent/files/upstream/lib/repair.sh index ddd243b5..a9b26d6c 100644 --- a/autosysadmin-agent/files/upstream/lib/repair.sh +++ b/autosysadmin-agent/files/upstream/lib/repair.sh @@ -92,9 +92,9 @@ repair_lxc_php() { lxc-start --daemon --name "${container_name}" rc=$? if [ "${rc}" -eq "0" ]; then - log_all "Restart LXC container '${container_name}: OK" + log_action "Restart LXC container '${container_name}: OK" else - log_all "Restart LXC container '${container_name}: failed" + log_action "Restart LXC container '${container_name}: failed" fi # Save LXC info (after restart) From 23f4f9690fa8eaf87d14c083a85831ffd6d48632 Mon Sep 17 00:00:00 2001 From: David Prevot Date: Tue, 19 Mar 2024 16:53:35 +0100 Subject: [PATCH 13/40] roundcube: Use /var/log/roundcube directly /var/lib/roundcube/logs is already a symlink to /var/log/roundcube. --- fail2ban/templates/jail.local.j2 | 2 +- webapps/roundcube/templates/apache2.conf.j2 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban/templates/jail.local.j2 b/fail2ban/templates/jail.local.j2 index a1247f69..04259e4b 100644 --- a/fail2ban/templates/jail.local.j2 +++ b/fail2ban/templates/jail.local.j2 @@ -59,7 +59,7 @@ bantime = {{ fail2ban_wordpress_soft_bantime }} enabled = {{ fail2ban_roundcube }} port = http, https filter = roundcube -logpath = /var/lib/roundcube/logs/errors +logpath = /var/log/roundcube/errors maxretry = {{ fail2ban_roundcube_maxretry }} findtime = {{ fail2ban_roundcube_findtime }} bantime = {{ fail2ban_roundcube_bantime }} diff --git a/webapps/roundcube/templates/apache2.conf.j2 b/webapps/roundcube/templates/apache2.conf.j2 index 87bdf79e..db11b1ef 100644 --- a/webapps/roundcube/templates/apache2.conf.j2 +++ b/webapps/roundcube/templates/apache2.conf.j2 @@ -20,8 +20,8 @@ # LOG CustomLog /var/log/apache2/access.log vhost_combined - CustomLog /var/lib/roundcube/logs/access.log combined - ErrorLog /var/lib/roundcube/logs/error.log + CustomLog /var/log/roundcube/access.log combined + ErrorLog /var/log/roundcube/error.log # REWRITE UseCanonicalName On From 96c1017b5d92d73248318caad598493c556ec959 Mon Sep 17 00:00:00 2001 From: David Prevot Date: Tue, 19 Mar 2024 17:07:36 +0100 Subject: [PATCH 14/40] roundcube: Use /var/log/roundcube directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /home/roundcube/log does not even exists… --- webapps/roundcube/templates/apache2.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapps/roundcube/templates/apache2.conf.j2 b/webapps/roundcube/templates/apache2.conf.j2 index db11b1ef..0b926953 100644 --- a/webapps/roundcube/templates/apache2.conf.j2 +++ b/webapps/roundcube/templates/apache2.conf.j2 @@ -40,7 +40,7 @@ #php_admin_value upload_max_filesize 8M #php_admin_flag allow_url_fopen Off php_admin_value sendmail_path "/usr/sbin/sendmail -t -i -f www-roundcube" - php_admin_value error_log "/home/roundcube/log/php.log" + php_admin_value error_log "/var/log/roundcube/php.log" #php_admin_value open_basedir "/usr/share/php:/home/roundcube:/tmp" From fe66ad9c4f51b18c54461e2fba528d5dc216309b Mon Sep 17 00:00:00 2001 From: David Prevot Date: Tue, 19 Mar 2024 17:44:03 +0100 Subject: [PATCH 15/40] Changelog entry for roundcube log change --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97cac326..92c55ed8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The **patch** part is incremented if multiple releases happen the same month * autosysadmin-agent: upstream release 24.03.2 * evolinux-base: Add new variable to disable global customisation of bash config +* roundcube: Use /var/log/roundcube directly ### Fixed From b30b7c884a5b4b0b20bbb9de3409ac5b18d79f89 Mon Sep 17 00:00:00 2001 From: William Hirigoyen Date: Tue, 19 Mar 2024 18:09:56 +0100 Subject: [PATCH 16/40] fail2ban: SQLite purge script didn't vacuum as expected + error when vacuum cannot be done --- CHANGELOG.md | 1 + fail2ban/templates/fail2ban_dbpurge.j2 | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92c55ed8..d79fc5e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The **patch** part is incremented if multiple releases happen the same month * certbot: Fix HAProxy renewal hook * keepalived: Fix tasks that use file instead of copy * memcached: Fix conditions not properly writen (installation was always in multi-instance mode) +* fail2ban: SQLite purge script didn't vacuum as expected + error when vacuum cannot be done ### Removed diff --git a/fail2ban/templates/fail2ban_dbpurge.j2 b/fail2ban/templates/fail2ban_dbpurge.j2 index 26c80f32..c287ba99 100644 --- a/fail2ban/templates/fail2ban_dbpurge.j2 +++ b/fail2ban/templates/fail2ban_dbpurge.j2 @@ -2,12 +2,15 @@ # Juin - Decembre 2022 : #64088 # Purge pour Stretch et Buster -/usr/bin/ionice -c3 /usr/bin/sqlite3 /var/lib/fail2ban/fail2ban.sqlite3 "DELETE FROM bans WHERE datetime('now', '-{{ fail2ban_dbpurgeage_default }}') > datetime(timeofban, 'unixepoch');" +/usr/bin/ionice -c3 /usr/bin/sqlite3 /var/lib/fail2ban/fail2ban.sqlite3 ".timeout 5000; DELETE FROM bans WHERE datetime('now', '-{{ fail2ban_dbpurgeage_default }}') > datetime(timeofban, 'unixepoch');" -place_dispo=$( df -h /var/lib/fail2ban/fail2ban.sqlite3 --output="avail" -h --block-size=1 |tail -n1 ) -place_pris=$( echo $(("$(stat --format %s /var/lib/fail2ban/fail2ban.sqlite3 ) * 2" )) ) +place_dispo="$(df /var/lib/fail2ban/fail2ban.sqlite3 --output="avail" --block-size=1 | tail -n1)" +place_pris="$(stat --format %s /var/lib/fail2ban/fail2ban.sqlite3)" -if [ $place_pris -lt $place_dispo ] +if [ "$place_pris" -lt "$place_dispo" ] then /usr/bin/ionice -c3 /usr/bin/sqlite3 /var/lib/fail2ban/fail2ban.sqlite3 "VACUUM;" +else + >&2 echo "Fail2ban SQLite VACUUM not done because /var lacks of space (VACUUM may use twice the database size)." + exit 1 fi From 56eef89084ed6f346ac8bd7d645f2d1444fa78db Mon Sep 17 00:00:00 2001 From: William Hirigoyen Date: Fri, 22 Mar 2024 11:09:06 +0100 Subject: [PATCH 17/40] nagios-nrpe: create /etc/bash_completion.d if missing --- CHANGELOG.md | 1 + nagios-nrpe/tasks/check-local.yml | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d79fc5e4..756c881d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The **patch** part is incremented if multiple releases happen the same month * keepalived: Fix tasks that use file instead of copy * memcached: Fix conditions not properly writen (installation was always in multi-instance mode) * fail2ban: SQLite purge script didn't vacuum as expected + error when vacuum cannot be done +* nagios-nrpe: create /etc/bash_completion.d if missing ### Removed diff --git a/nagios-nrpe/tasks/check-local.yml b/nagios-nrpe/tasks/check-local.yml index d28d9113..69409314 100644 --- a/nagios-nrpe/tasks/check-local.yml +++ b/nagios-nrpe/tasks/check-local.yml @@ -15,10 +15,16 @@ dest: /usr/local/bin/check-local mode: "0755" -- name: Package bash-completion is intalled +- name: Package bash-completion is installed ansible.builtin.apt: name: bash-completion +- name: Directory /etc/bash_completion.d exists + ansible.builtin.file: + path: '/etc/bash_completion.d' + state: directory + mode: '0644' + - name: Completion for utilitary check-local is installed ansible.builtin.copy: src: check-local_completion From 96504b1deb89fb876589edacb80e4978e343f631 Mon Sep 17 00:00:00 2001 From: Alexis Ben Miloud--Josselin Date: Wed, 27 Mar 2024 12:13:49 +0100 Subject: [PATCH 18/40] evolinux-users: Add sudo mvcli for nagios user --- CHANGELOG.md | 1 + evolinux-users/templates/sudoers.j2 | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 756c881d..be066a67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The **patch** part is incremented if multiple releases happen the same month * autosysadmin-agent: upstream release 24.03.2 * evolinux-base: Add new variable to disable global customisation of bash config * roundcube: Use /var/log/roundcube directly +* evolinux-users: Add sudo mvcli for nagios user ### Fixed diff --git a/evolinux-users/templates/sudoers.j2 b/evolinux-users/templates/sudoers.j2 index 0e8471bf..b6510ed9 100644 --- a/evolinux-users/templates/sudoers.j2 +++ b/evolinux-users/templates/sudoers.j2 @@ -24,6 +24,8 @@ nagios ALL = NOPASSWD: /sbin/megacli -LdInfo -Lall -aALL -NoLog nagios ALL = NOPASSWD: /sbin/megacli -AdpBbuCmd -GetBbuStatus -aALL -NoLog nagios ALL = NOPASSWD: /sbin/ssacli controller all show status nagios ALL = NOPASSWD: /sbin/ssacli controller slot=0 logicaldrive all show +nagios ALL = NOPASSWD: /usr/local/bin/mvcli info -o blk +nagios ALL = NOPASSWD: /usr/local/bin/mvcli info -o vd nagios ALL = NOPASSWD: /usr/local/lib/nagios/plugins/check_gluster.rb nagios ALL = (clamav) NOPASSWD: /usr/bin/clamscan /tmp/safe.txt From 5acb1956f5a088b5ac371c70ed989050f8e5e0ab Mon Sep 17 00:00:00 2001 From: William Hirigoyen Date: Wed, 27 Mar 2024 15:17:48 +0100 Subject: [PATCH 19/40] packweb: fix old bug (2017!) .orig file created by module patch and taken in account by ProFTPd --- CHANGELOG.md | 1 + webapps/evoadmin-web/tasks/ftp.yml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be066a67..82b4e7fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The **patch** part is incremented if multiple releases happen the same month * memcached: Fix conditions not properly writen (installation was always in multi-instance mode) * fail2ban: SQLite purge script didn't vacuum as expected + error when vacuum cannot be done * nagios-nrpe: create /etc/bash_completion.d if missing +* packweb: fix old bug (2017!) .orig file created by module patch and taken in account by ProFTPd ### Removed diff --git a/webapps/evoadmin-web/tasks/ftp.yml b/webapps/evoadmin-web/tasks/ftp.yml index 8c400e68..614ac4f0 100644 --- a/webapps/evoadmin-web/tasks/ftp.yml +++ b/webapps/evoadmin-web/tasks/ftp.yml @@ -10,3 +10,9 @@ remote_src: False src: ftp/evolinux.conf.diff dest: /etc/proftpd/conf.d/z-evolinux.conf + +- name: Remove .orig file created by previous patch task + ansible.builtin.file: + path: /etc/proftpd/conf.d/z-evolinux.conf.orig + state: absent + From 7a9be8d6fad7063c84127feae5a2648e48d73e94 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 4 Apr 2024 18:40:39 +0200 Subject: [PATCH 20/40] vrrpd : configure and restart minifirewall before starting VRRP --- CHANGELOG.md | 1 + vrrpd/tasks/ip.yml | 49 +++++++++++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82b4e7fa..af3c13c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The **patch** part is incremented if multiple releases happen the same month * evolinux-base: Add new variable to disable global customisation of bash config * roundcube: Use /var/log/roundcube directly * evolinux-users: Add sudo mvcli for nagios user +* vrrpd : configure and restart minifirewall before starting VRRP ### Fixed diff --git a/vrrpd/tasks/ip.yml b/vrrpd/tasks/ip.yml index 81c9f08f..4f951928 100644 --- a/vrrpd/tasks/ip.yml +++ b/vrrpd/tasks/ip.yml @@ -1,25 +1,6 @@ --- -- name: set unit name - ansible.builtin.set_fact: - vrrp_systemd_unit_name: "vrrp-{{ vrrp_address.id }}.service" - -- name: add systemd unit - ansible.builtin.template: - src: vrrp.service.j2 - dest: "/etc/systemd/system/{{ vrrp_systemd_unit_name }}" - force: true - register: vrrp_systemd_unit - -- name: enable and start systemd unit - ansible.builtin.systemd: - name: "{{ vrrp_systemd_unit_name }}" - daemon_reload: yes - enabled: yes - state: "{{ vrrp_address.state }}" - when: - - vrrp_systemd_unit is changed - - not ansible_check_mode +# Configure and restart minifirewall before starting the VRRP service - name: Check if a recent minifirewall is present ansible.builtin.stat: @@ -55,3 +36,31 @@ loop_var: peer notify: "{{ minifirewall_restart_handler_name }}" when: _minifirewall_dir.stat.exists + +- name: Flush handlers to restart minifirewall + ansible.builtin.meta: flush_handlers + when: _minifirewall_dir.stat.exists + + +# Configure VRRP service + +- name: set unit name + ansible.builtin.set_fact: + vrrp_systemd_unit_name: "vrrp-{{ vrrp_address.id }}.service" + +- name: add systemd unit + ansible.builtin.template: + src: vrrp.service.j2 + dest: "/etc/systemd/system/{{ vrrp_systemd_unit_name }}" + force: true + register: vrrp_systemd_unit + +- name: enable and start systemd unit + ansible.builtin.systemd: + name: "{{ vrrp_systemd_unit_name }}" + daemon_reload: yes + enabled: yes + state: "{{ vrrp_address.state }}" + when: + - vrrp_systemd_unit is changed + - not ansible_check_mode From 4bbe2f4f72939e0cf168b6dc6510977336a5c41d Mon Sep 17 00:00:00 2001 From: Eric Morino Date: Tue, 9 Apr 2024 09:12:01 +0200 Subject: [PATCH 21/40] Delete 'state' option on template for PGDG repo --- postgresql/tasks/pgdg-repo.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/postgresql/tasks/pgdg-repo.yml b/postgresql/tasks/pgdg-repo.yml index 7df2a722..223ceb29 100644 --- a/postgresql/tasks/pgdg-repo.yml +++ b/postgresql/tasks/pgdg-repo.yml @@ -36,7 +36,6 @@ ansible.builtin.template: src: postgresql.sources.j2 dest: /etc/apt/sources.list.d/postgresql.sources - state: present register: postgresql_sources when: ansible_distribution_major_version is version('12', '>=') From 5708e7205d1d9fd1d006282f5450546882e552b4 Mon Sep 17 00:00:00 2001 From: Brice Waegeneire Date: Thu, 11 Apr 2024 15:48:37 +0200 Subject: [PATCH 22/40] nrpe: !disk1 exclude filesystem type overlay --- CHANGELOG.md | 1 + nagios-nrpe/templates/evolix.cfg.j2 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af3c13c8..2fb17809 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The **patch** part is incremented if multiple releases happen the same month * roundcube: Use /var/log/roundcube directly * evolinux-users: Add sudo mvcli for nagios user * vrrpd : configure and restart minifirewall before starting VRRP +* nrpe: !disk1 exclude filesystem type overlay ### Fixed diff --git a/nagios-nrpe/templates/evolix.cfg.j2 b/nagios-nrpe/templates/evolix.cfg.j2 index d725bb3b..79b01f66 100644 --- a/nagios-nrpe/templates/evolix.cfg.j2 +++ b/nagios-nrpe/templates/evolix.cfg.j2 @@ -9,7 +9,7 @@ allowed_hosts={{ nagios_nrpe_allowed_hosts | join(',') }} # System checks command[check_load]=/usr/lib/nagios/plugins/check_load --percpu --warning=0.7,0.6,0.5 --critical=0.9,0.8,0.7 command[check_swap]=/usr/lib/nagios/plugins/check_swap -a -w 30% -c 20% -command[check_disk1]=/usr/lib/nagios/plugins/check_disk -e -w 10% -c 3% -W 10% -K 3% -C -w 5% -c 2% -W 5% -K 2% -p /home -x /lib/init/rw -x /dev -x /dev/shm -x /run -I '^/run/' -I '^/sys/' +command[check_disk1]=/usr/lib/nagios/plugins/check_disk -e -w 10% -c 3% -W 10% -K 3% -C -w 5% -c 2% -W 5% -K 2% -p /home -x /lib/init/rw -x /dev -x /dev/shm -x /run -I '^/run/' -I '^/sys/' -X overlay command[check_zombie_procs]=sudo /usr/lib/nagios/plugins/check_procs -w 5 -c 10 -s Z command[check_total_procs]=sudo /usr/lib/nagios/plugins/check_procs -w 400 -c 600 command[check_users]=/usr/lib/nagios/plugins/check_users -w 5 -c 10 From 2a264dd2bccd71060afb0e277ccd7839ab0edaa0 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 12 Apr 2024 15:54:20 +0200 Subject: [PATCH 23/40] redis: replace inline argument with environment variable for the password --- CHANGELOG.md | 1 + nagios-nrpe/files/plugins/check_sentinel | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fb17809..e53b4e80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The **patch** part is incremented if multiple releases happen the same month * fail2ban: SQLite purge script didn't vacuum as expected + error when vacuum cannot be done * nagios-nrpe: create /etc/bash_completion.d if missing * packweb: fix old bug (2017!) .orig file created by module patch and taken in account by ProFTPd +* redis: replace inline argument with environment variable for the password ### Removed diff --git a/nagios-nrpe/files/plugins/check_sentinel b/nagios-nrpe/files/plugins/check_sentinel index a76e45e7..6d28b120 100755 --- a/nagios-nrpe/files/plugins/check_sentinel +++ b/nagios-nrpe/files/plugins/check_sentinel @@ -104,7 +104,7 @@ redis_cli_args='' sentinel_port=$(awk '/^port/{print $2}' "${sentinel_config_file}") ! test -z "$sentinel_port" && redis_cli_args="${redis_cli_args} -p ${sentinel_port}" sentinel_pass=$(awk '/^requirepass/{print $2}' "${sentinel_config_file}") -! test -z "$sentinel_pass" && redis_cli_args="${redis_cli_args} --pass ${sentinel_pass}" +! test -z "$sentinel_pass" && export REDISCLI_AUTH="${sentinel_pass}" alias _redis-cli="redis-cli ${redis_cli_args}" # List all masters names known by sentinel From d11cf4987ba7af7a76478bf78e09e728cd4bc2b3 Mon Sep 17 00:00:00 2001 From: Mathieu Gauthier-Pilote Date: Mon, 8 Apr 2024 11:17:43 -0400 Subject: [PATCH 24/40] Add option to return the amp values --- munin/files/plugins/ipmi_ | 224 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 munin/files/plugins/ipmi_ diff --git a/munin/files/plugins/ipmi_ b/munin/files/plugins/ipmi_ new file mode 100644 index 00000000..51e713d3 --- /dev/null +++ b/munin/files/plugins/ipmi_ @@ -0,0 +1,224 @@ +#!/bin/bash +# -*- sh -*- + +: << =cut + +=head1 NAME + +ipmi_ - Plugin to monitor temperature, fan speed, watts or volts using IPMI + +=head1 CONFIGURATION + +=head2 ENVIRONMENT VARIABLES + +This plugin does not use environment variables + +=head2 WILDCARD PLUGIN + +This plugin should be linked as ipmi_temp, ipmi_fans, ipmi_power or ipmi_volts, +and will show either temperatures, fan speeds, watts or volts based on its link +name. + +=head1 NOTE + +WARNING: Munin has a 10 second default timeout on plugins. On some +hosts ipmitool takes longer than that to probe all your hardware. In +this case this plugin us unusable. + +=head1 AUTHOR + +Nicolai Langfeldt + +Modified by Mathieu Gauthier-Pilote from Evolix to return amp values as well (2024/04). + +=head1 LICENSE + +Donated to the public domain by Nicolai Langfeldt (janl@linpro.no) + +=head1 MAGIC MARKERS + + #%# family=auto + #%# capabilities=autoconf suggest + +=cut + +#### Parse commandline to determine what the job is + +CONFIG=no + +case $1 in + autoconf) + type -p ipmitool &>/dev/null || + { echo 'no (missing ipmitool command)' && exit 0; } + + ipmitool sensor &>/dev/null || + { echo 'no (unable to access IPMI device)' && exit 0; } + + echo yes + exit 0 + ;; + suggest) echo fans + echo temp + echo power + echo volts + echo amp + exit 0;; + config) CONFIG=config;; +esac + +case $0 in + *_temp) MEASURE=temp;; + *_fans) MEASURE=fans;; + *_power) MEASURE=power;; + *_volts) MEASURE=volts;; + *_amp) MEASURE=amp;; + *) echo "Please invoke as ipmi_temp, ipmi_fans, ipmi_power ipmi_volts or ipmi_amp" >&2 + exit 1;; +esac + +export CONFIG MEASURE + +#### Work is done in this awk script + +ipmitool sensor | gawk -F'|' ' +BEGIN { + FANS = ""; + TEMPS = ""; + POWER = ""; + VOLTS = ""; + AMP = ""; + CFANS = "graph_title Fan speeds based on IPMI\ngraph_vlabel RPM or %\ngraph_category Sensors\n"; + CTEMPS = "graph_title Machine temperature based on IPMI\ngraph_vlabel Degrees celcius\ngraph_category Sensors\n"; + CPOWER = "graph_title Power usage based on IPMI\ngraph_vlabel W\ngraph_category Sensors\n"; + CVOLTS = "graph_title Volts based on IPMI\ngraph_vlabel V\ngraph_category Sensors\n"; + CAMP = "graph_title Amps based on IPMI\ngraph_vlabel A\ngraph_category Sensors\n"; +} + +# Remove extraneous spaces to make output prettyer +{ gsub(/\t/," "); gsub(/ +/," "); gsub(/ +\|/,"|"); gsub(/\| +/,"|") } + +# Skip lines with 0x0 in first column +/^[^|]+\|0x0\|/ { next; }; + +# Skip lines with na in first column +/^[^|]+\|na\|/ { next; }; + +# Parse temperatures +/degrees C/ { + NAME=THING=$1; + gsub(/[^A-Za-z0-9]/,"",NAME); + TEMP=$2; + + # Find unique name + while (NAMES[NAME] >= 1) { + NAME=sprintf("%si",NAME); + } + NAMES[NAME]=1; + + WARN=$8; + CRIT=$9; + + TEMPS = sprintf("%s%s.value %s\n",TEMPS,NAME,TEMP); + CTEMPS = sprintf("%s%s.label %s\n",CTEMPS,NAME,THING); + + if (CRIT !~ /na/) { + CTEMPS = sprintf("%s%s.critical 0:%s\n",CTEMPS,NAME,CRIT); + } + + if (WARN !~ /na/) { + CTEMPS = sprintf("%s%s.warning 0:%s\n",CTEMPS,NAME,WARN); + } +} + +/(RPM|^Fan.*percent)/ { + NAME=THING=$1; + gsub(/[^A-Za-z0-9]/,"",NAME); + SPEED=$2; + + # Find unique name + while (NAMES[NAME] >= 1) { + NAME=sprintf("%si",NAME); + } + NAMES[NAME]=1; + + FANS = sprintf("%s%s.value %s\n",FANS,NAME,SPEED); + CFANS = sprintf("%s%s.label %s\n",CFANS,NAME,THING); + + OK=$4; + + MIN=$6; + if (MIN !~ /na/) { + CFANS = sprintf("%s%s.warning %s:\n",CFANS,NAME,MIN); + } +} + +/Watts/ { + NAME=THING=$1; + gsub(/[^A-Za-z0-9]/,"",NAME); + WATTS=$2; + + # Find unique name + while (NAMES[NAME] >= 1) { + NAME=sprintf("%si",NAME); + } + NAMES[NAME]=1; + + POWER = sprintf("%s%s.value %s\n",POWER,NAME,WATTS); + CPOWER = sprintf("%s%s.label %s\n",CPOWER,NAME,THING); +} + +/Volts/ { + NAME=THING=$1 + gsub(/[^A-Za-z0-9]/,"",NAME); + VOLTS_SENSOR=$2; + + # Find unique name + while (NAMES[NAME] >= 1) { + NAME=sprintf("%si",NAME); + } + NAMES[NAME]=1; + + VOLTS = sprintf("%s%s.value %s\n",VOLTS,NAME,VOLTS_SENSOR); + CVOLTS = sprintf("%s%s.label %s\n",CVOLTS,NAME,THING); +} + +/Amps/ { + NAME=THING=$1 + gsub(/[^A-Za-z0-9]/,"",NAME); + AMPS=$2; + + # Find unique name + while (NAMES[NAME] >= 1) { + NAME=sprintf("%si",NAME); + } + NAMES[NAME]=1; + + AMP = sprintf("%s%s.value %s\n",AMP,NAME,AMPS); + CAMP = sprintf("%s%s.label %s\n",CAMP,NAME,THING); +} + +END { + if (ENVIRON["MEASURE"] == "temp") { + VALUE=TEMPS; + CONFIG=CTEMPS; + } else if (ENVIRON["MEASURE"] == "power") { + VALUE=POWER; + CONFIG=CPOWER; + } else if (ENVIRON["MEASURE"] == "volts") { + VALUE=VOLTS; + CONFIG=CVOLTS; + } else if (ENVIRON["MEASURE"] == "amp") { + VALUE=AMP; + CONFIG=CAMP; + } else { + VALUE=FANS; + CONFIG=CFANS; + } + if (ENVIRON["CONFIG"] == "config") + printf "%s",CONFIG; + else + printf "%s",VALUE; +} +' + +# vim: syntax=sh ts=4 et From d1410e38a1a42a28ac4ce3be280aa607d942e274 Mon Sep 17 00:00:00 2001 From: William Hirigoyen Date: Tue, 16 Apr 2024 10:25:13 +0200 Subject: [PATCH 25/40] evolinux-base/logcheck: fix conf patch, journal check was not disabled when asked --- CHANGELOG.md | 1 + evolinux-base/tasks/logs.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e53b4e80..5138471d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The **patch** part is incremented if multiple releases happen the same month * nagios-nrpe: create /etc/bash_completion.d if missing * packweb: fix old bug (2017!) .orig file created by module patch and taken in account by ProFTPd * redis: replace inline argument with environment variable for the password +* evolinux-base/logcheck: fix conf patch, journal check was not disabled when asked ### Removed diff --git a/evolinux-base/tasks/logs.yml b/evolinux-base/tasks/logs.yml index 46ada1d2..b8838c9e 100644 --- a/evolinux-base/tasks/logs.yml +++ b/evolinux-base/tasks/logs.yml @@ -66,7 +66,7 @@ # Logcheck - name: Disable logcheck monitoring of journald ansible.builtin.lineinfile: - dest: /etc/logrotate.conf + dest: /etc/logcheck/logcheck.logfiles.d/journal.logfiles line: "#journal" regexp: "^journal" when: evolinux_logs_disable_logcheck_journald | bool From 0ec343766d3a3f04f793a2612a4e51281326c56e Mon Sep 17 00:00:00 2001 From: William Hirigoyen Date: Wed, 17 Apr 2024 17:09:12 +0200 Subject: [PATCH 26/40] postfix/amavis: max servers is now 3 (previously 2) --- CHANGELOG.md | 1 + amavis/templates/amavis.conf.j2 | 2 +- postfix/templates/packmail_master.cf.j2 | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5138471d..c98d3e9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The **patch** part is incremented if multiple releases happen the same month * evolinux-users: Add sudo mvcli for nagios user * vrrpd : configure and restart minifirewall before starting VRRP * nrpe: !disk1 exclude filesystem type overlay +* postfix/amavis: max servers is now 3 (previously 2) ### Fixed diff --git a/amavis/templates/amavis.conf.j2 b/amavis/templates/amavis.conf.j2 index 8bc9bae8..210c4890 100644 --- a/amavis/templates/amavis.conf.j2 +++ b/amavis/templates/amavis.conf.j2 @@ -39,7 +39,7 @@ $sa_spam_subject_tag = '[SPAM]'; $log_level = 2; # En fonction besoin/ressources, on a juste le nbre de process -$max_servers = 2; +$max_servers = 3; $enable_ldap = 1; $default_ldap = { diff --git a/postfix/templates/packmail_master.cf.j2 b/postfix/templates/packmail_master.cf.j2 index 9627fcb3..f3910238 100644 --- a/postfix/templates/packmail_master.cf.j2 +++ b/postfix/templates/packmail_master.cf.j2 @@ -134,7 +134,7 @@ localhost:10026 inet n - y - 10 smtpd -o smtpd_recipient_restrictions=permit_mynetworks,reject -o mynetworks=127.0.0.0/8 -smtp-amavis unix - - y - 2 lmtp +smtp-amavis unix - - y - 3 lmtp -o lmtp_data_done_timeout=1200 -o lmtp_send_xforward_command=yes From f4e6aabe8aa6f42091888eb20d03f42501e14778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Dubois?= Date: Thu, 18 Apr 2024 11:03:45 +0200 Subject: [PATCH 27/40] openvpn: install packages manually Because openbsd_pkg module is broken since OpenBSD 7.4 with the version of Ansible we currently use --- CHANGELOG.md | 1 + openvpn/tasks/openbsd.yml | 30 +++++++++++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c98d3e9c..0dff2c1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ The **patch** part is incremented if multiple releases happen the same month * packweb: fix old bug (2017!) .orig file created by module patch and taken in account by ProFTPd * redis: replace inline argument with environment variable for the password * evolinux-base/logcheck: fix conf patch, journal check was not disabled when asked +* openvpn: install packages manually, because openbsd_pkg module is broken since OpenBSD 7.4 with the version of Ansible we currently use ### Removed diff --git a/openvpn/tasks/openbsd.yml b/openvpn/tasks/openbsd.yml index 28781880..820e3b20 100644 --- a/openvpn/tasks/openbsd.yml +++ b/openvpn/tasks/openbsd.yml @@ -1,9 +1,16 @@ --- +- name: Check if OpenVPN is already installed + ansible.builtin.command: + cmd: pkg_info -Iq inst:openvpn + register: is_installed + ignore_errors: true + changed_when: false + - name: Install OpenVPN - community.general.openbsd_pkg: - name: openvpn-- - when: ansible_distribution == 'OpenBSD' + ansible.builtin.command: + cmd: pkg_add openvpn-- + when: "'Can\\'t find inst:' in is_installed.stderr" - name: Create /etc/openvpn ansible.builtin.file: @@ -116,10 +123,19 @@ check_mode: no register: nrpe_evolix_config -- name: Install NRPE check dependencies - community.general.openbsd_pkg: - name: p5-Net-Telnet - when: nrpe_evolix_config.stat.exists +- name: Check if NRPE check dependenciy is already installed + ansible.builtin.command: + cmd: pkg_info -Iq inst:p5-Net-Telnet + register: is_installed + ignore_errors: true + changed_when: false + +- name: Install p5-Net-Telnet + ansible.builtin.command: + cmd: pkg_add p5-Net-Telnet + when: + - "'Can\\'t find inst:' in is_installed.stderr" + - nrpe_evolix_config.stat.exists - name: Install OpenVPN NRPE check ansible.builtin.copy: From 16394060c9cab85986f63b0c1a26faac1e71ca28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Dubois?= Date: Thu, 18 Apr 2024 11:11:03 +0200 Subject: [PATCH 28/40] openvpn: let tasks using openbsd_pkg but commented --- openvpn/tasks/openbsd.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/openvpn/tasks/openbsd.yml b/openvpn/tasks/openbsd.yml index 820e3b20..e9c0685e 100644 --- a/openvpn/tasks/openbsd.yml +++ b/openvpn/tasks/openbsd.yml @@ -1,5 +1,10 @@ --- +# openbsd_pkg is broken since OpenBSD 7.4 with the version of Ansible we currently use +#- name: Install OpenVPN +# community.general.openbsd_pkg: +# name: openvpn-- + - name: Check if OpenVPN is already installed ansible.builtin.command: cmd: pkg_info -Iq inst:openvpn @@ -123,14 +128,20 @@ check_mode: no register: nrpe_evolix_config -- name: Check if NRPE check dependenciy is already installed +# openbsd_pkg is broken since OpenBSD 7.4 with the version of Ansible we currently use +#- name: Install NRPE check dependency +# community.general.openbsd_pkg: +# name: p5-Net-Telnet +# when: nrpe_evolix_config.stat.exists + +- name: Check if NRPE check dependency is already installed ansible.builtin.command: cmd: pkg_info -Iq inst:p5-Net-Telnet register: is_installed ignore_errors: true changed_when: false -- name: Install p5-Net-Telnet +- name: Install NRPE check dependency ansible.builtin.command: cmd: pkg_add p5-Net-Telnet when: From 9a65312190e695dfcbdbc1ae95ca74f3a8a3bfc1 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 18 Apr 2024 15:10:01 +0200 Subject: [PATCH 29/40] evolinux-base: disable logcheck monitoring of journald only if journald.logfiles exists --- CHANGELOG.md | 1 + evolinux-base/tasks/logs.yml | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dff2c1c..78d41498 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The **patch** part is incremented if multiple releases happen the same month * autosysadmin-agent: upstream release 24.03.2 * evolinux-base: Add new variable to disable global customisation of bash config +* evolinux-base: Disable logcheck monitoring of journald only if journald.logfiles exists * roundcube: Use /var/log/roundcube directly * evolinux-users: Add sudo mvcli for nagios user * vrrpd : configure and restart minifirewall before starting VRRP diff --git a/evolinux-base/tasks/logs.yml b/evolinux-base/tasks/logs.yml index b8838c9e..ce796d6c 100644 --- a/evolinux-base/tasks/logs.yml +++ b/evolinux-base/tasks/logs.yml @@ -64,12 +64,19 @@ when: evolinux_logs_default_dateext | bool # Logcheck +- name: Check if journald.logfiles exists + stat: + path: /etc/logcheck/logcheck.logfiles.d/journal.logfiles + register: _logcheck_journald_logfiles + - name: Disable logcheck monitoring of journald ansible.builtin.lineinfile: dest: /etc/logcheck/logcheck.logfiles.d/journal.logfiles line: "#journal" regexp: "^journal" - when: evolinux_logs_disable_logcheck_journald | bool + when: + - _logcheck_journald_logfiles.stat.exists + - evolinux_logs_disable_logcheck_journald | bool # Journald - name: /etc/systemd/journald.conf.d/ is present From f8e92d2eebfed40375ff2dc2e39f9fb2876cdcac Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 18 Apr 2024 15:11:34 +0200 Subject: [PATCH 30/40] haproxy: support bookworm for backport packages --- CHANGELOG.md | 5 +++-- haproxy/defaults/main.yml | 3 ++- haproxy/tasks/main.yml | 1 - haproxy/tasks/packages_backports.yml | 4 ++++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78d41498..a54666fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,11 +18,12 @@ The **patch** part is incremented if multiple releases happen the same month * autosysadmin-agent: upstream release 24.03.2 * evolinux-base: Add new variable to disable global customisation of bash config * evolinux-base: Disable logcheck monitoring of journald only if journald.logfiles exists -* roundcube: Use /var/log/roundcube directly * evolinux-users: Add sudo mvcli for nagios user -* vrrpd : configure and restart minifirewall before starting VRRP +* haproxy: support bookworm for backport packages * nrpe: !disk1 exclude filesystem type overlay * postfix/amavis: max servers is now 3 (previously 2) +* roundcube: Use /var/log/roundcube directly +* vrrpd : configure and restart minifirewall before starting VRRP ### Fixed diff --git a/haproxy/defaults/main.yml b/haproxy/defaults/main.yml index 50f6bb48..f0c1652e 100644 --- a/haproxy/defaults/main.yml +++ b/haproxy/defaults/main.yml @@ -35,5 +35,6 @@ haproxy_deny_ips: [] haproxy_backports_packages_stretch: haproxy libssl1.0.0 haproxy_backports_packages_buster: haproxy haproxy_backports_packages_bullseye: haproxy +haproxy_backports_packages_bookworm: haproxy -haproxy_allow_ip_nonlocal_bind: Null \ No newline at end of file +haproxy_allow_ip_nonlocal_bind: Null diff --git a/haproxy/tasks/main.yml b/haproxy/tasks/main.yml index 12fdd224..c8c38f05 100644 --- a/haproxy/tasks/main.yml +++ b/haproxy/tasks/main.yml @@ -21,7 +21,6 @@ - name: Self-signed certificate is present in HAProxy ssl directory ansible.builtin.shell: cmd: "cat /etc/ssl/certs/ssl-cert-snakeoil.pem /etc/ssl/private/ssl-cert-snakeoil.key > /etc/haproxy/ssl/ssl-cert-snakeoil.pem" - args: creates: /etc/haproxy/ssl/ssl-cert-snakeoil.pem notify: reload haproxy tags: diff --git a/haproxy/tasks/packages_backports.yml b/haproxy/tasks/packages_backports.yml index 2a5a855c..6ad4e325 100644 --- a/haproxy/tasks/packages_backports.yml +++ b/haproxy/tasks/packages_backports.yml @@ -19,6 +19,10 @@ haproxy_backports_packages: "{{ haproxy_backports_packages_bullseye }}" when: ansible_distribution_release == 'bullseye' +- ansible.builtin.set_fact: + haproxy_backports_packages: "{{ haproxy_backports_packages_bookworm }}" + when: ansible_distribution_release == 'bookworm' + - name: Prefer HAProxy package from backports ansible.builtin.template: src: haproxy_apt_preferences.j2 From 42ad242aaf1c5914972a88cde13e15c95cbe1fce Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 18 Apr 2024 15:18:42 +0200 Subject: [PATCH 31/40] vrrpd: configure minifirewall with blocks instead of lines --- CHANGELOG.md | 3 ++- vrrpd/defaults/main.yml | 5 ++++- vrrpd/tasks/ip.yml | 35 +++++++++++++++++++++++------------ 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a54666fb..719424ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,8 @@ The **patch** part is incremented if multiple releases happen the same month * nrpe: !disk1 exclude filesystem type overlay * postfix/amavis: max servers is now 3 (previously 2) * roundcube: Use /var/log/roundcube directly -* vrrpd : configure and restart minifirewall before starting VRRP +* vrrpd: configure and restart minifirewall before starting VRRP +* vrrpd: configure minifirewall with blocks instead of lines ### Fixed diff --git a/vrrpd/defaults/main.yml b/vrrpd/defaults/main.yml index 1c7abb10..f55b54c6 100644 --- a/vrrpd/defaults/main.yml +++ b/vrrpd/defaults/main.yml @@ -9,9 +9,12 @@ vrrp_addresses: [] # priority: Null # the priority of this host in the virtual server (default: 100) # authentication: Null # authentification type: auth=(none|pw/hexkey|ah/hexkey) hexkey=0x[0-9a-fA-F]+ # label: Null # use this name is syslog messages (helps when several vrid are running) -# ip: Null # the ip address(es) (and optionnaly subnet mask) of the virtual server +# ip: Null # the IP address(es) (and optionnaly subnet mask) of the virtual server +# peers: [IP1, IP2] # list of peers (IP), for minifirewall rules # state: Null # 'started' or 'stopped' # } +vrrp_manage_minifirewall: true + minifirewall_restart_if_needed: True minifirewall_restart_force: False diff --git a/vrrpd/tasks/ip.yml b/vrrpd/tasks/ip.yml index 4f951928..a7b645cb 100644 --- a/vrrpd/tasks/ip.yml +++ b/vrrpd/tasks/ip.yml @@ -11,35 +11,46 @@ minifirewall_restart_handler_name: "{{ minifirewall_restart_if_needed | bool | ternary('restart minifirewall', 'restart minifirewall (noop)') }}" - name: VRRP output is authorized in minifirewall - lineinfile: + ansible.builtin.blockinfile: path: /etc/minifirewall.d/vrrpd - line: "/sbin/iptables -A OUTPUT -o {{ vrrp_address.interface }} -p 112 -j ACCEPT # Allow VRRP output on {{ vrrp_address.interface }}" - regexp: "# Allow VRRP output on {{ vrrp_address.interface }}$" + marker: "## {mark} ANSIBLE MANAGED OUTPUT RULES FOR VRID {{ vrrp_address.id }}" + block: | + /sbin/iptables -A OUTPUT -o {{ vrrp_address.interface }} -p 112 -j ACCEPT # Allow VRRP output on {{ vrrp_address.interface }} create: yes mode: "0600" owner: "root" group: "root" notify: "{{ minifirewall_restart_handler_name }}" - when: _minifirewall_dir.stat.exists + when: + - vrrp_manage_minifirewall | bool + - _minifirewall_dir.stat.exists - name: VRRP input is authorized in minifirewall - lineinfile: + ansible.builtin.blockinfile: path: /etc/minifirewall.d/vrrpd - line: "/sbin/iptables -A INPUT -i {{ vrrp_address.interface }} -s {{ peer }} -d 224.0.0.0/8 -j ACCEPT # Allow VRRP input on {{ vrrp_address.interface }} from {{ peer }} for VRID {{ vrrp_address.id }}" - regexp: "# Allow VRRP input on {{ vrrp_address.interface }} from {{ peer }} for VRID {{ vrrp_address.id }}" + marker: "## {mark} ANSIBLE MANAGED INPUT RULES FOR VRID {{ vrrp_address.id }}" + block: | + {% if vrrp_address.peers | default([]) | length <= 0 %} + /sbin/iptables -A INPUT -i {{ vrrp_address.interface }} -d 224.0.0.0/8 -j ACCEPT # Allow VRRP input on {{ vrrp_address.interface }} for VRID {{ vrrp_address.id }} + {% else %} + {% for peer in vrrp_address.peers %} + /sbin/iptables -A INPUT -i {{ vrrp_address.interface }} -s {{ peer }} -d 224.0.0.0/8 -j ACCEPT # Allow VRRP input on {{ vrrp_address.interface }} from {{ peer }} for VRID {{ vrrp_address.id }} + {% endfor %} + {% endif %} create: yes mode: "0600" owner: "root" group: "root" - loop: "{{ vrrp_address.peers | default([]) }}" - loop_control: - loop_var: peer notify: "{{ minifirewall_restart_handler_name }}" - when: _minifirewall_dir.stat.exists + when: + - vrrp_manage_minifirewall | bool + - _minifirewall_dir.stat.exists - name: Flush handlers to restart minifirewall ansible.builtin.meta: flush_handlers - when: _minifirewall_dir.stat.exists + when: + - vrrp_manage_minifirewall | bool + - _minifirewall_dir.stat.exists # Configure VRRP service From 8cd887ee21208ad2b972fac6d0fcdb12791a1ff6 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 18 Apr 2024 15:18:58 +0200 Subject: [PATCH 32/40] minifirewall: fix task name --- minifirewall/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/minifirewall/tasks/main.yml b/minifirewall/tasks/main.yml index 5457d60c..23fe7e48 100644 --- a/minifirewall/tasks/main.yml +++ b/minifirewall/tasks/main.yml @@ -74,7 +74,7 @@ ####################################################################### -- name: Fail if minifirewall_main_file is defined (legacy mode) +- name: Fail if minifirewall_main_file is defined (modern mode) ansible.builtin.fail: msg: "Variable minifirewall_main_file is deprecated and not configurable anymore." when: @@ -179,4 +179,4 @@ - always when: - minifirewall_install_mode != 'legacy' - - minifirewall_restart_force | bool \ No newline at end of file + - minifirewall_restart_force | bool From a41e78b556f8166deaaba7a228bd0e6609a85685 Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Thu, 18 Apr 2024 15:38:11 +0200 Subject: [PATCH 33/40] docker-host: Removed setting docker_conf_use_iptables (iptable usage forced to true --- CHANGELOG.md | 2 ++ docker-host/defaults/main.yml | 3 --- docker-host/tasks/main.yml | 9 +++++++++ docker-host/templates/daemon.json.j2 | 6 +----- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 719424ec..dacad144 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,8 @@ The **patch** part is incremented if multiple releases happen the same month ### Removed +* docker-host: Removed setting docker_conf_use_iptables (iptable usage forced to true) + ### Security ## [24.03] 2024-03-01 diff --git a/docker-host/defaults/main.yml b/docker-host/defaults/main.yml index bc5dc88f..ac93e596 100644 --- a/docker-host/defaults/main.yml +++ b/docker-host/defaults/main.yml @@ -3,9 +3,6 @@ docker_home: /var/lib/docker docker_tmpdir: "{{ docker_home }}/tmp" -# Chose to use iptables instead of docker-proxy userland process -docker_conf_use_iptables: False - # Disable the possibility for containers processes to gain new privileges docker_conf_no_newprivileges: False diff --git a/docker-host/tasks/main.yml b/docker-host/tasks/main.yml index ec3781e7..d35b7d7d 100644 --- a/docker-host/tasks/main.yml +++ b/docker-host/tasks/main.yml @@ -1,5 +1,14 @@ # This role installs the docker daemon --- + +- name: Fail if docker_conf_use_iptables is defined + ansible.builtin.fail: + msg: "Variable docker_conf_use_iptables is deprecated and not configurable anymore. Please remove it from your variables. Also double-check the daemon.json config for docker" + when: + - docker_conf_use_iptables is defined + tags: + - always + - name: Remove older docker packages ansible.builtin.apt: name: diff --git a/docker-host/templates/daemon.json.j2 b/docker-host/templates/daemon.json.j2 index 92d60f8d..e1f6d6b1 100644 --- a/docker-host/templates/daemon.json.j2 +++ b/docker-host/templates/daemon.json.j2 @@ -1,5 +1,6 @@ { "debug": false + ,"iptables": true {# Docker data-dir (default to /var/lib/docker) #} ,"data-root": "{{ docker_home }}" {# Keep containers running while docker daemon downtime #} @@ -7,11 +8,6 @@ {% if docker_conf_user_namespace %} {# Turn on user namespace remaping #} ,"userns-remap": "default" -{% endif %} -{% if docker_conf_use_iptables %} - {# Use iptables instead of docker-proxy #} - ,"userland-proxy": false - ,"iptables": true {% endif %} {# Disable the possibility for containers processes to gain new privileges #} ,"no-new-privileges": {{ docker_conf_no_newprivileges | to_json }} From 5d114683272ff838b7d4dfac89050b56a51cba5b Mon Sep 17 00:00:00 2001 From: Ludovic Poujol Date: Thu, 18 Apr 2024 16:10:26 +0200 Subject: [PATCH 34/40] docker-host: lint --- docker-host/defaults/main.yml | 14 +++++++------- docker-host/tasks/main.yml | 22 ++++++++++++---------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/docker-host/defaults/main.yml b/docker-host/defaults/main.yml index ac93e596..f2dc0e9e 100644 --- a/docker-host/defaults/main.yml +++ b/docker-host/defaults/main.yml @@ -4,24 +4,24 @@ docker_home: /var/lib/docker docker_tmpdir: "{{ docker_home }}/tmp" # Disable the possibility for containers processes to gain new privileges -docker_conf_no_newprivileges: False +docker_conf_no_newprivileges: false # Toggle live restore (need to be disabled in swarm mode) -docker_conf_live_restore: True +docker_conf_live_restore: true # Toggle user namespace -docker_conf_user_namespace: True +docker_conf_user_namespace: true # Disable all default network connectivity -docker_conf_disable_default_networking: False +docker_conf_disable_default_networking: false # Remote access -docker_remote_access_enabled: False +docker_remote_access_enabled: false docker_daemon_port: 2376 docker_daemon_listening_ip: 0.0.0.0 # TLS -docker_tls_enabled: False +docker_tls_enabled: false docker_tls_path: "{{ docker_home }}/tls" docker_tls_ca: ca/ca.pem docker_tls_ca_key: ca/ca-key.pem @@ -29,4 +29,4 @@ docker_tls_cert: server/cert.pem docker_tls_key: server/key.pem docker_tls_csr: server/server.csr -apt_keyring_dir: "{{ ansible_distribution_major_version is version('12', '<') | ternary('/etc/apt/trusted.gpg.d', '/etc/apt/keyrings') }}" \ No newline at end of file +apt_keyring_dir: "{{ ansible_distribution_major_version is version('12', '<') | ternary('/etc/apt/trusted.gpg.d', '/etc/apt/keyrings') }}" diff --git a/docker-host/tasks/main.yml b/docker-host/tasks/main.yml index d35b7d7d..f36dc457 100644 --- a/docker-host/tasks/main.yml +++ b/docker-host/tasks/main.yml @@ -32,7 +32,7 @@ when: ansible_distribution_major_version is version('10', '<') - name: "Ensure {{ apt_keyring_dir }} directory exists" - file: + ansible.builtin.file: path: "{{ apt_keyring_dir }}" state: directory mode: "755" @@ -53,35 +53,34 @@ repo: 'deb [signed-by={{ apt_keyring_dir }}/docker-debian.asc] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable' filename: docker state: present - update_cache: yes + update_cache: true when: ansible_distribution_major_version is version('12', '<') - name: Add Docker repository (Debian >=12) ansible.builtin.template: src: docker.sources.j2 dest: /etc/apt/sources.list.d/docker.sources - register: docker_sources + owner: root + group: root + mode: "0644" when: ansible_distribution_major_version is version('12', '>=') -- name: Update APT cache - ansible.builtin.apt: - update_cache: yes - when: docker_sources is changed - - name: Install Docker ansible.builtin.apt: name: - docker-ce - docker-ce-cli - containerd.io + update_cache: true + cache_valid_time: 3600 -- name: python-docker is installed +- name: Package python-docker is installed ansible.builtin.apt: name: python-docker state: present when: ansible_python_version is version('3', '<') -- name: python3-docker is installed +- name: Package python3-docker is installed ansible.builtin.apt: name: python3-docker state: present @@ -91,6 +90,9 @@ ansible.builtin.template: src: daemon.json.j2 dest: /etc/docker/daemon.json + owner: root + group: root + mode: "0644" notify: restart docker - name: Creating Docker tmp directory From 4c2548b21d312a1b0f4e9a3d5559133b5f6b8cfe Mon Sep 17 00:00:00 2001 From: Mathieu Trossevin Date: Fri, 26 Apr 2024 09:29:15 +0200 Subject: [PATCH 35/40] fix(certbot): Fix HAPEE renewal hook --- CHANGELOG.md | 1 + certbot/files/hooks/deploy/hapee.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dacad144..753556e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ The **patch** part is incremented if multiple releases happen the same month ### Fixed * certbot: Fix HAProxy renewal hook +* certbot: Fix HAPEE renewal hook * keepalived: Fix tasks that use file instead of copy * memcached: Fix conditions not properly writen (installation was always in multi-instance mode) * fail2ban: SQLite purge script didn't vacuum as expected + error when vacuum cannot be done diff --git a/certbot/files/hooks/deploy/hapee.sh b/certbot/files/hooks/deploy/hapee.sh index d39da25b..8d0e234f 100644 --- a/certbot/files/hooks/deploy/hapee.sh +++ b/certbot/files/hooks/deploy/hapee.sh @@ -40,7 +40,7 @@ concat_files() { } cert_and_key_mismatch() { hapee_cert_md5=$(openssl x509 -noout -pubkey -in "${hapee_cert_file}" | openssl md5) - hapee_key_md5=$(openssl pkey -noout -pubout -in "${hapee_cert_file}" | openssl md5) + hapee_key_md5=$(openssl pkey -pubout -in "${hapee_cert_file}" | openssl md5) test "${hapee_cert_md5}" != "${hapee_key_md5}" } From e5bbb601d37a27ff895d3aad3a0bf73cc0606c9f Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 30 Apr 2024 17:08:44 +0200 Subject: [PATCH 36/40] evobackup-client: replace non-functional role with install tasks --- CHANGELOG.md | 2 + evobackup-client/README.md | 27 +- evobackup-client/defaults/main.yml | 27 +- evobackup-client/files/upstream/CHANGELOG.md | 82 + .../files/upstream/bin/evobackupctl | 153 ++ .../files/upstream/lib/dump/elasticsearch.sh | 301 ++++ .../files/upstream/lib/dump/misc.sh | 559 ++++++ .../files/upstream/lib/dump/mysql.sh | 1551 +++++++++++++++++ .../files/upstream/lib/dump/postgresql.sh | 343 ++++ evobackup-client/files/upstream/lib/main.sh | 466 +++++ .../files/upstream/lib/utilities.sh | 143 ++ .../files/upstream/lib/zzz_evobackup.sh | 326 ++++ evobackup-client/tasks/install.yml | 50 + evobackup-client/tasks/main.yml | 45 +- .../templates/update-evobackup-canary.sh.j2 | 3 + .../templates/zzz_evobackup.default.sh.j2 | 305 ---- 16 files changed, 4031 insertions(+), 352 deletions(-) create mode 100644 evobackup-client/files/upstream/CHANGELOG.md create mode 100644 evobackup-client/files/upstream/bin/evobackupctl create mode 100644 evobackup-client/files/upstream/lib/dump/elasticsearch.sh create mode 100644 evobackup-client/files/upstream/lib/dump/misc.sh create mode 100644 evobackup-client/files/upstream/lib/dump/mysql.sh create mode 100644 evobackup-client/files/upstream/lib/dump/postgresql.sh create mode 100644 evobackup-client/files/upstream/lib/main.sh create mode 100644 evobackup-client/files/upstream/lib/utilities.sh create mode 100644 evobackup-client/files/upstream/lib/zzz_evobackup.sh create mode 100644 evobackup-client/tasks/install.yml create mode 100644 evobackup-client/templates/update-evobackup-canary.sh.j2 delete mode 100644 evobackup-client/templates/zzz_evobackup.default.sh.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 753556e1..1b98d4d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ The **patch** part is incremented if multiple releases happen the same month ### Changed * autosysadmin-agent: upstream release 24.03.2 +* evobackup-client: replace non-functional role with install tasks +* evobackup-client: upstream release 24.04.1 * evolinux-base: Add new variable to disable global customisation of bash config * evolinux-base: Disable logcheck monitoring of journald only if journald.logfiles exists * evolinux-users: Add sudo mvcli for nagios user diff --git a/evobackup-client/README.md b/evobackup-client/README.md index 18ef132e..8d0b770c 100644 --- a/evobackup-client/README.md +++ b/evobackup-client/README.md @@ -1,23 +1,16 @@ # evobackup-client -Allows the configuration of backups to one or more remote filesystems. +Install the necessary libraries and script to configure backup scripts. -The backup hosts and the ports in use need to be defined in -evobackup-client__hosts before running it. +Additional information: -The default zzz_evobackup.sh configures a system only backup, but the -template can be overriden to configure a full backup instead. If -you change the variables in defaults/main.yml you can easily run -this again and configure backups to a second set of hosts. +* [evobackup-client documentation](https://gitea.evolix.org/evolix/evobackup/src/branch/master/client/README.md) +* canary -Do not forget to set the evobackup-client__mail variable to an -email adress you control. +## Available variables -You can add this example to an installation playbook to create the -ssh key without running the rest of the role. - -~~~ - post_tasks: - - include_role: - name: evobackup-client tasks_from: ssh_key.yml -~~~ +* `evobackup_client__lib_dir` : directory for libraries (default: `/usr/local/lib/evobackup`) +* `evobackup_client__bin_dir` : directory for scripts/binaries (default: `/usr/local/bin`) +* `evobackup_client__update_canary_enable` : should the canary be updated (default: `True`) +* `evobackup_client__update_canary_path` : path for the canary update script (default: `/etc/cron.daily/000-update-evobackup-canary`) +* `evobackup_client__update_canary_who` : who the canary update must be attributed to (default: `@daily`) diff --git a/evobackup-client/defaults/main.yml b/evobackup-client/defaults/main.yml index 56c0e5d8..20eaa069 100644 --- a/evobackup-client/defaults/main.yml +++ b/evobackup-client/defaults/main.yml @@ -1,15 +1,22 @@ --- -evobackup_client__root_key_path: "/root/.ssh/id_ed25519" -evobackup_client__root_key_type: "ed25519" -evobackup_client__cron_path: "/etc/cron.daily/zzz_evobackup" -evobackup_client__cron_template_name: "zzz_evobackup" -evobackup_client__mail: null -evobackup_client__servers_fallback: -1 -evobackup_client__pid_path: "/var/run/evobackup.pid" -evobackup_client__log_path: "/var/log/evobackup.log" -evobackup_client__backup_path: "/home/backup" -evobackup_client__hosts: null +# evobackup_client__root_key_path: "/root/.ssh/id_ed25519" +# evobackup_client__root_key_type: "ed25519" +# evobackup_client__cron_path: "/etc/cron.daily/zzz_evobackup" +# evobackup_client__cron_template_name: "zzz_evobackup" +# evobackup_client__mail: null +# evobackup_client__servers_fallback: -1 +# evobackup_client__pid_path: "/var/run/evobackup.pid" +# evobackup_client__log_path: "/var/log/evobackup.log" +# evobackup_client__backup_path: "/home/backup" +# evobackup_client__hosts: null # - name: "backups.example.org" # ip: "xxx.xxx.xxx.xxx" # fingerprint: "ecdsa-sha2-nistp256 ..." # port: xxxx + +evobackup_client__lib_dir: "/usr/local/lib/evobackup" +evobackup_client__bin_dir: "/usr/local/bin" + +evobackup_client__update_canary_enable: True +evobackup_client__update_canary_path: /etc/cron.daily/000-update-evobackup-canary +evobackup_client__update_canary_who: "@daily" diff --git a/evobackup-client/files/upstream/CHANGELOG.md b/evobackup-client/files/upstream/CHANGELOG.md new file mode 100644 index 00000000..9d5c1681 --- /dev/null +++ b/evobackup-client/files/upstream/CHANGELOG.md @@ -0,0 +1,82 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). + +This project does not follow semantic versioning. +The **major** part of the version is the year +The **minor** part changes is the month +The **patch** part changes is incremented if multiple releases happen the same month + +## [Unreleased] + +### Added + +### Changed + +### Deprecated + +### Removed + +### Fixed + +### Security + +## [24.04.1] + +### Fixed + +* evobackupctl: quote ARGS variable for options parsing. + +## [24.04] + +### Added + +* Vagrant definition for manual tests + +### Changed + +* split functions into libraries +* add evobackupctl script +* change the "zzz_evobackup" script to a template, easy to copy with evobackupctl +* use env-based shebang for shell scripts +* use $TMPDIR if available + +### Removed + +* update-evobackup-canary is managed by ansible-roles.git +* deployment by Ansible is managed elsewhere (now in evolix-private.git, later in ansible-roles.git) + +### Fixed + +* don't exit the whole program if a sync task can't be done + +## [22.12] + +### Changed + +* Use --dump-dir instead of --backup-dir to suppress dump-server-state warning +* Do not use rsync compression +* Replace rsync option --verbose by --itemize-changes +* Add canary to zzz_evobackup +* update-evobackup-canary: do not use GNU date, for it to be compatible with OpenBSD +* Add AGPL License and README +* Script now depends on Bash +* tolerate absence of mtr or traceroute +* Only one loop for all Redis instances +* remodel how we build the rsync command +* use sub shells instead of moving around +* Separate Rsync for the canary file if the main Rsync has finished without errors + +### Removed + +* No more fallback if dump-server-state is missing + +### Fixed + +* Make start_time and stop_time compatible with OpenBSD + +## [22.03] + +Split client and server parts of the project diff --git a/evobackup-client/files/upstream/bin/evobackupctl b/evobackup-client/files/upstream/bin/evobackupctl new file mode 100644 index 00000000..33996945 --- /dev/null +++ b/evobackup-client/files/upstream/bin/evobackupctl @@ -0,0 +1,153 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2155 +readonly PROGNAME=$(basename "${0}") +# shellcheck disable=SC2155 +readonly PROGDIR=$(readlink -m "$(dirname "${0}")") +# shellcheck disable=SC2124 +readonly ARGS=$@ + +# Change this to wherever you install the libraries +readonly LIBDIR="/usr/local/lib/evobackup" + +source "${LIBDIR}/main.sh" + +show_version() { + cat <, + Jérémy Lecour . + +${PROGNAME} comes with ABSOLUTELY NO WARRANTY. This is free software, +and you are welcome to redistribute it under certain conditions. +See the GNU General Public License v3.0 for details. +END +} +show_help() { + cat < /dev/null || hostname -I | awk '{ print $1}') + fi + + echo "Copy-paste those lines on backup server(s) :" + echo "----------" + echo "SERVER_NAME=${SERVER_NAME}" + echo "SERVER_IP=${SERVER_IP}" + echo "echo '${SSH_KEY}' > /root/\${SERVER_NAME}.pub" + echo "bkctld init \${SERVER_NAME}" + echo "bkctld key \${SERVER_NAME} /root/\${SERVER_NAME}.pub" + echo "bkctld ip \${SERVER_NAME} \${SERVER_IP}" + echo "bkctld start \${SERVER_NAME}" + echo "bkctld status \${SERVER_NAME}" + echo "grep --quiet --extended-regexp \"^\\s?NODE=\" /etc/default/bkctld && bkctld sync \${SERVER_NAME}" + echo "----------" +} + +copy_template() { + dest_path=${1} + dest_dir="$(dirname "${dest_path}")" + + if [ -e "${dest_path}" ]; then + printf "Path for new evobackup script '%s' already exists.\n" "${dest_path}" >&2 + exit 1 + elif [ ! -e "${dest_dir}" ]; then + printf "Parent directory '%s' doesn't exist. Create it first.\n" "${dest_dir}" >&2 + exit 1 + else + if cp "${LIBDIR}/zzz_evobackup.sh" "${dest_path}"; then + chmod 750 "${dest_path}" + + sed -i "s|@COMMAND@|${PROGDIR}/${PROGNAME} ${ARGS}|" "${dest_path}" + sed -i "s|@DATE@|$(date --iso-8601=seconds)|" "${dest_path}" + sed -i "s|@VERSION@|${VERSION}|" "${dest_path}" + + printf "New evobackup script has been saved to '%s'.\n" "${dest_path}" + printf "Remember to customize it (mail notifications, backup servers…).\n" + exit 0 + fi + fi +} + +main() { + # If no argument is provided, print help and exit + # shellcheck disable=SC2086 + if [ -z "${ARGS}" ]; then + show_help + exit 0 + fi + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case ${1:-''} in + -V|--version) + show_version + exit 0 + ;; + -h|--help) + show_help + exit 0 + ;; + --jail-init-commands) + jail_init_commands + exit 0 + ;; + --copy-template) + # copy-template option, with value separated by space + if [ -n "$2" ]; then + copy_template "${2}" + shift + else + printf "'%s' requires a non-empty option argument.\n" "--copy-template" >&2 + exit 1 + fi + ;; + --copy-template=?*) + # copy-template option, with value separated by = + copy_template "${1#*=}" + ;; + --copy-template=) + # copy-template option, without value + printf "'%s' requires a non-empty option argument.\n" "--copy-template" >&2 + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + printf "unknown option '%s'.\n" "${1}" >&2 + exit 1 + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done +} + +main ${ARGS} diff --git a/evobackup-client/files/upstream/lib/dump/elasticsearch.sh b/evobackup-client/files/upstream/lib/dump/elasticsearch.sh new file mode 100644 index 00000000..1e5475bb --- /dev/null +++ b/evobackup-client/files/upstream/lib/dump/elasticsearch.sh @@ -0,0 +1,301 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2034,SC2317,SC2155 + +####################################################################### +# Snapshot Elasticsearch data +# +# Arguments: +# --protocol= (default: http) +# --cacert=[String] (default: ) +# path to the CA certificate to use when using https +# --host=[String] (default: localhost) +# --port=[Integer] (default: 9200) +# --user=[String] (default: ) +# --password=[String] (default: ) +# --repository=[String] (default: snaprepo) +# --snapshot=[String] (default: snapshot.daily) +####################################################################### +dump_elasticsearch() { + local option_protocol="http" + local option_cacert="" + local option_host="localhost" + local option_port="9200" + local option_user="" + local option_password="" + local option_repository="snaprepo" + local option_snapshot="snapshot.daily" + local option_others="" + + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case ${1:-''} in + --protocol) + # protocol options, with value separated by space + if [ -n "$2" ]; then + option_protocol="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--protocol' requires a non-empty option argument." + exit 1 + fi + ;; + --protocol=?*) + # protocol options, with value separated by = + option_protocol="${1#*=}" + ;; + --protocol=) + # protocol options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--protocol' requires a non-empty option argument." + exit 1 + ;; + --cacert) + # cacert options, with value separated by space + if [ -n "$2" ]; then + option_cacert="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--cacert' requires a non-empty option argument." + exit 1 + fi + ;; + --cacert=?*) + # cacert options, with value separated by = + option_cacert="${1#*=}" + ;; + --cacert=) + # cacert options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--cacert' requires a non-empty option argument." + exit 1 + ;; + --host) + # host options, with value separated by space + if [ -n "$2" ]; then + option_host="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--host' requires a non-empty option argument." + exit 1 + fi + ;; + --host=?*) + # host options, with value separated by = + option_host="${1#*=}" + ;; + --host=) + # host options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--host' requires a non-empty option argument." + exit 1 + ;; + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + ;; + --user) + # user options, with value separated by space + if [ -n "$2" ]; then + option_user="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + fi + ;; + --user=?*) + # user options, with value separated by = + option_user="${1#*=}" + ;; + --user=) + # user options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + ;; + --password) + # password options, with value separated by space + if [ -n "$2" ]; then + option_password="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + fi + ;; + --password=?*) + # password options, with value separated by = + option_password="${1#*=}" + ;; + --password=) + # password options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + ;; + --repository) + # repository options, with value separated by space + if [ -n "$2" ]; then + option_repository="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--repository' requires a non-empty option argument." + exit 1 + fi + ;; + --repository=?*) + # repository options, with value separated by = + option_repository="${1#*=}" + ;; + --repository=) + # repository options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--repository' requires a non-empty option argument." + exit 1 + ;; + --snapshot) + # snapshot options, with value separated by space + if [ -n "$2" ]; then + option_snapshot="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--snapshot' requires a non-empty option argument." + exit 1 + fi + ;; + --snapshot=?*) + # snapshot options, with value separated by = + option_snapshot="${1#*=}" + ;; + --snapshot=) + # snapshot options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--snapshot' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + option_others=${*} + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: unknown option '${1}' (ignored)" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + # Use the default Elasticsearch CA certificate when using HTTPS, if not specified directly + local default_cacert="/etc/elasticsearch/certs/http_ca.crt" + if [ "${option_protocol}" = "https" ] && [ -z "${option_cacert}" ] && [ -f "${default_cacert}" ]; then + option_cacert="${default_cacert}" + fi + + local errors_dir="${ERRORS_DIR}/elasticsearch-${option_repository}-${option_snapshot}" + rm -rf "${errors_dir}" + mkdir -p "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${errors_dir}" + + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${option_snapshot}" + + ## Take a snapshot as a backup. + ## Warning: You need to have a path.repo configured. + ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes + + local base_url="${option_protocol}://${option_host}:${option_port}" + local repository_url="${base_url}/_snapshot/${option_repository}" + local snapshot_url="${repository_url}/${option_snapshot}" + + # Verify snapshot repository + + local error_file="${errors_dir}/verify.err" + + declare -a connect_options + connect_options=() + if [ -n "${option_cacert}" ]; then + connect_options+=(--cacert "${option_cacert}") + fi + if [ -n "${option_user}" ] || [ -n "${option_password}" ]; then + local connect_options+=("--user ${option_user}:${option_password}") + fi + if [ -n "${option_others}" ]; then + # word splitting is deliberate here + # shellcheck disable=SC2206 + connect_options+=(${option_others}) + fi + # Add the http return code at the end of the output + connect_options+=(--write-out '%{http_code}\n') + connect_options+=(--silent) + + declare -a dump_options + dump_options=() + dump_options+=(--request POST) + + dump_cmd="curl ${connect_options[*]} ${dump_options[*]} ${repository_url}/_verify?pretty" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} > "${error_file}" + + # test if the last line of the log file is "200" + tail -n 1 "${error_file}" | grep --quiet "^200$" "${error_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: repository verification returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + + # Delete snapshot + + declare -a dump_options + dump_options=() + dump_options+=(--request DELETE) + + dump_cmd="curl ${connect_options[*]} ${dump_options[*]} ${snapshot_url}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} > /dev/null + + # Create snapshot + + local error_file="${errors_dir}/create.err" + + declare -a dump_options + dump_options=() + dump_options+=(--request PUT) + + dump_cmd="curl ${connect_options[*]} ${dump_options[*]} ${snapshot_url}?wait_for_completion=true" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} > "${error_file}" + + # test if the last line of the log file is "200" + tail -n 1 "${error_file}" | grep --quiet "^200$" "${error_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: curl returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + fi + + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${option_snapshot}" +} diff --git a/evobackup-client/files/upstream/lib/dump/misc.sh b/evobackup-client/files/upstream/lib/dump/misc.sh new file mode 100644 index 00000000..c7ea1fa6 --- /dev/null +++ b/evobackup-client/files/upstream/lib/dump/misc.sh @@ -0,0 +1,559 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2034,SC2317,SC2155 + +####################################################################### +# Dump LDAP files (config, data, all) +# +# Arguments: +####################################################################### +dump_ldap() { + ## OpenLDAP : example with slapcat + local dump_dir="${LOCAL_BACKUP_DIR}/ldap" + rm -rf "${dump_dir}" + mkdir -p "${dump_dir}" + chmod 700 "${dump_dir}" + + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${FUNCNAME[0]} to ${dump_dir}" + + dump_cmd="slapcat -n 0 -l ${dump_dir}/config.bak" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} + + dump_cmd="slapcat -n 1 -l ${dump_dir}/data.bak" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} + + dump_cmd="slapcat -l ${dump_dir}/all.bak" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} + + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${FUNCNAME[0]}" +} + +####################################################################### +# Copy dump file of Redis instances +# +# Arguments: +# --instances=[Integer] (default: all) +####################################################################### +dump_redis() { + all_instances=$(find /var/lib/ -mindepth 1 -maxdepth 1 '(' -type d -o -type l ')' -name 'redis*') + + local option_instances="" + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case ${1:-''} in + --instances) + # instances options, with key and value separated by space + if [ -n "$2" ]; then + if [ "${2}" == "all" ]; then + read -a option_instances <<< "${all_instances}" + else + IFS="," read -a option_instances <<< "${2}" + fi + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--instances' requires a non-empty option argument." + exit 1 + fi + ;; + --instances=?*) + # instances options, with key and value separated by = + if [ "${1#*=}" == "all" ]; then + read -a option_instances <<< "${all_instances}" + else + IFS="," read -a option_instances <<< "${1#*=}" + fi + ;; + --instances=) + # instances options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--instances' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: unknown option '${1}' (ignored)" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + for instance in "${option_instances[@]}"; do + name=$(basename "${instance}") + local dump_dir="${LOCAL_BACKUP_DIR}/${name}" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + if [ -f "${instance}/dump.rdb" ]; then + local error_file="${errors_dir}/${name}.err" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_dir}" + + # Copy the Redis database + dump_cmd="cp -a ${instance}/dump.rdb ${dump_dir}/dump.rdb" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} 2> "${error_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: cp ${instance}/dump.rdb to ${dump_dir} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + + # Compress the Redis database + dump_cmd="gzip ${dump_dir}/dump.rdb" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: gzip ${dump_dir}/dump.rdb returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_dir}" + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '${instance}/dump.rdb' not found." + fi + done +} + +####################################################################### +# Dump all collections of a MongoDB database +# using a custom authentication, instead of /etc/mysql/debian.cnf +# +# Arguments: +# --port=[String] (default: ) +# --user=[String] (default: ) +# --password=[String] (default: ) +# --dump-label=[String] (default: "default") +# used as suffix of the dump dir to differenciate multiple instances +# Other options after -- are passed as-is to mongodump +# +# don't forget to create use with read-only access +# > use admin +# > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } ) +####################################################################### +dump_mongodb() { + local option_port="" + local option_user="" + local option_password="" + local option_dump_label="" + local option_others="" + + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case ${1:-''} in + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + ;; + --user) + # user options, with value separated by space + if [ -n "$2" ]; then + option_user="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + fi + ;; + --user=?*) + # user options, with value separated by = + option_user="${1#*=}" + ;; + --user=) + # user options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + ;; + --password) + # password options, with value separated by space + if [ -n "$2" ]; then + option_password="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + fi + ;; + --password=?*) + # password options, with value separated by = + option_password="${1#*=}" + ;; + --password=) + # password options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + ;; + --dump-label) + # dump-label options, with value separated by space + if [ -n "$2" ]; then + option_dump_label="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + fi + ;; + --dump-label=?*) + # dump-label options, with value separated by = + option_dump_label="${1#*=}" + ;; + --dump-label=) + # dump-label options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + option_others=${*} + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: unknown option '${1}' (ignored)" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + if [ -z "${option_dump_label}" ]; then + if [ -n "${option_port}" ]; then + option_dump_label="${option_port}" + else + option_dump_label="default" + fi + fi + + local dump_dir="${LOCAL_BACKUP_DIR}/mongodb-${option_dump_label}" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + local error_file="${errors_dir}.err" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_dir}" + + declare -a dump_options + dump_options=() + if [ -n "${option_port}" ]; then + dump_options+=(--port="${option_port}") + fi + if [ -n "${option_user}" ]; then + dump_options+=(--username="${option_user}") + fi + if [ -n "${option_password}" ]; then + dump_options+=(--password="${option_password}") + fi + dump_options+=(--out="${dump_dir}/") + if [ -n "${option_others}" ]; then + # word splitting is deliberate here + # shellcheck disable=SC2206 + dump_options+=(${option_others}) + fi + + dump_cmd="mongodump ${dump_options[*]}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd} > /dev/null" + ${dump_cmd} 2> "${error_file}" > /dev/null + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mongodump to ${dump_dir} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - stop ${FUNCNAME[0]}: ${dump_dir}" +} + +####################################################################### +# Dump RAID configuration +# +# Arguments: +####################################################################### +dump_raid_config() { + local dump_dir="${LOCAL_BACKUP_DIR}/raid" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + if command -v megacli > /dev/null; then + local error_file="${errors_dir}/megacli.cfg" + local dump_file="${dump_dir}/megacli.err" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + dump_cmd="megacli -CfgSave -f ${dump_file} -a0" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} 2> "${error_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: megacli to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" + elif command -v perccli > /dev/null; then + local error_file="${errors_dir}/perccli.cfg" + local dump_file="${dump_dir}/perccli.err" + # log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + # TODO: find out what the correct command is + + # dump_cmd="perccli XXXX" + # log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + # ${dump_cmd} 2> ${error_file} + + # local last_rc=$? + # # shellcheck disable=SC2086 + # if [ ${last_rc} -ne 0 ]; then + # log_error "LOCAL_TASKS - ${FUNCNAME[0]}: perccli to ${dump_file} returned an error ${last_rc}" "${error_file}" + # GLOBAL_RC=${E_DUMPFAILED} + # else + # rm -f "${error_file}" + # fi + # log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" + else + log "LOCAL_TASKS - ${FUNCNAME[0]}: 'megacli' and 'perccli' not found, unable to dump RAID configuration" + fi +} + +####################################################################### +# Save some traceroute/mtr results +# +# Arguments: +# --targets=[IP,HOST] (default: ) +####################################################################### +dump_traceroute() { + local option_targets="" + + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case ${1:-''} in + --targets) + # targets options, with key and value separated by space + if [ -n "$2" ]; then + IFS="," read -a option_targets <<< "${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--targets' requires a non-empty option argument." + exit 1 + fi + ;; + --targets=?*) + # targets options, with key and value separated by = + IFS="," read -a option_targets <<< "${1#*=}" + ;; + --targets=) + # targets options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--targets' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: unknown option '${1}' (ignored)" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + local dump_dir="${LOCAL_BACKUP_DIR}/traceroute" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + + mtr_bin=$(command -v mtr) + if [ -n "${mtr_bin}" ]; then + for target in "${option_targets[@]}"; do + local dump_file="${dump_dir}/mtr-${target}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + ${mtr_bin} -r "${target}" > "${dump_file}" + + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" + done + fi + + traceroute_bin=$(command -v traceroute) + if [ -n "${traceroute_bin}" ]; then + for target in "${option_targets[@]}"; do + local dump_file="${dump_dir}/traceroute-${target}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + ${traceroute_bin} -n "${target}" > "${dump_file}" 2>&1 + + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" + done + fi +} + +####################################################################### +# Save many system information, using dump_server_state +# +# Arguments: +# any option for dump-server-state (except --dump-dir) is usable +# (default: --all) +####################################################################### +dump_server_state() { + local dump_dir="${LOCAL_BACKUP_DIR}/server-state" + rm -rf "${dump_dir}" + # Do not create the directory + # mkdir -p -m 700 "${dump_dir}" + + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_dir}" + + # pass all options + read -a options <<< "${@}" + # if no option is given, use "--all" as fallback + if [ ${#options[@]} -le 0 ]; then + options=(--all) + fi + # add "--dump-dir" in case it is missing (as it should) + options+=(--dump-dir "${dump_dir}") + + dump_server_state_bin=$(command -v dump-server-state) + if [ -z "${dump_server_state_bin}" ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: dump-server-state is missing" + rc=1 + else + dump_cmd="${dump_server_state_bin} ${options[*]}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: dump-server-state returned an error ${last_rc}, check ${dump_dir}" + GLOBAL_RC=${E_DUMPFAILED} + fi + fi + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_dir}" +} + +####################################################################### +# Save RabbitMQ data +# +# Arguments: +# +# Warning: This has been poorly tested +####################################################################### +dump_rabbitmq() { + local dump_dir="${LOCAL_BACKUP_DIR}/rabbitmq" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + local error_file="${errors_dir}.err" + local dump_file="${dump_dir}/config" + + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + dump_cmd="rabbitmqadmin export ${dump_file}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} 2> "${error_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: pg_dump to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" +} + +####################################################################### +# Save Files ACL on various partitions. +# +# Arguments: +####################################################################### +dump_facl() { + local dump_dir="${LOCAL_BACKUP_DIR}/facl" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_dir}" + + dump_cmd="getfacl -R /etc > ${dump_dir}/etc.txt" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} + + dump_cmd="getfacl -R /home > ${dump_dir}/home.txt" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} + + dump_cmd="getfacl -R /usr > ${dump_dir}/usr.txt" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} + + dump_cmd="getfacl -R /var > ${dump_dir}/var.txt" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} + + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_dir}" +} diff --git a/evobackup-client/files/upstream/lib/dump/mysql.sh b/evobackup-client/files/upstream/lib/dump/mysql.sh new file mode 100644 index 00000000..0d288b40 --- /dev/null +++ b/evobackup-client/files/upstream/lib/dump/mysql.sh @@ -0,0 +1,1551 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2034,SC2317,SC2155 + +####################################################################### +# Dump complete summary of an instance (using pt-mysql-summary) +# +# Arguments: +# --port=[Integer] (default: ) +# --socket=[String] (default: ) +# --user=[String] (default: ) +# --password=[String] (default: ) +# --defaults-file=[String] (default: ) +# --defaults-extra-file=[String] (default: ) +# --defaults-group-suffix=[String] (default: ) +# --dump-label=[String] (default: "default") +# used as suffix of the dump dir to differenciate multiple instances +####################################################################### +dump_mysql_summary() { + local option_port="" + local option_socket="" + local option_defaults_file="" + local option_defaults_extra_file="" + local option_defaults_group_suffix="" + local option_user="" + local option_password="" + local option_dump_label="" + + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case ${1:-''} in + --defaults-file) + # defaults-file options, with value separated by space + if [ -n "$2" ]; then + option_defaults_file="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-file=?*) + # defaults-file options, with value separated by = + option_defaults_file="${1#*=}" + ;; + --defaults-file=) + # defaults-file options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + ;; + --defaults-extra-file) + # defaults-file options, with value separated by space + if [ -n "$2" ]; then + option_defaults_extra_file="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-extra-file=?*) + # defaults-extra-file options, with value separated by = + option_defaults_extra_file="${1#*=}" + ;; + --defaults-extra-file=) + # defaults-extra-file options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-extra-file' requires a non-empty option argument." + exit 1 + ;; + --defaults-group-suffix) + # defaults-group-suffix options, with value separated by space + if [ -n "$2" ]; then + option_defaults_group_suffix="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-group-suffix' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-group-suffix=?*) + # defaults-group-suffix options, with value separated by = + option_defaults_group_suffix="${1#*=}" + ;; + --defaults-group-suffix=) + # defaults-group-suffix options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-group-suffix' requires a non-empty option argument." + exit 1 + ;; + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + ;; + --socket) + # socket options, with value separated by space + if [ -n "$2" ]; then + option_socket="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--socket' requires a non-empty option argument." + exit 1 + fi + ;; + --socket=?*) + # socket options, with value separated by = + option_socket="${1#*=}" + ;; + --socket=) + # socket options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--socket' requires a non-empty option argument." + exit 1 + ;; + --user) + # user options, with value separated by space + if [ -n "$2" ]; then + option_user="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + fi + ;; + --user=?*) + # user options, with value separated by = + option_user="${1#*=}" + ;; + --user=) + # user options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + ;; + --password) + # password options, with value separated by space + if [ -n "$2" ]; then + option_password="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + fi + ;; + --password=?*) + # password options, with value separated by = + option_password="${1#*=}" + ;; + --password=) + # password options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + ;; + --dump-label) + # dump-label options, with value separated by space + if [ -n "$2" ]; then + option_dump_label="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + fi + ;; + --dump-label=?*) + # dump-label options, with value separated by = + option_dump_label="${1#*=}" + ;; + --dump-label=) + # dump-label options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: unkwnown option (ignored): '${1}'" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + if [ -z "${option_dump_label}" ]; then + if [ -n "${option_defaults_group_suffix}" ]; then + option_dump_label="${option_defaults_group_suffix}" + elif [ -n "${option_port}" ]; then + option_dump_label="${option_port}" + elif [ -n "${option_socket}" ]; then + option_dump_label=$(path_to_str "${option_socket}") + else + option_dump_label="default" + fi + fi + + local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}-summary" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + ## Dump all grants (requires 'percona-toolkit' package) + if command -v pt-mysql-summary > /dev/null; then + local error_file="${errors_dir}/mysql-summary.err" + local dump_file="${dump_dir}/mysql-summary.out" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + ## Connection options + declare -a connect_options + connect_options=() + if [ -n "${option_defaults_file}" ]; then + connect_options+=(--defaults-file="${option_defaults_file}") + fi + if [ -n "${option_defaults_extra_file}" ]; then + connect_options+=(--defaults-extra-file="${option_defaults_extra_file}") + fi + if [ -n "${option_defaults_group_suffix}" ]; then + connect_options+=(--defaults-group-suffix="${option_defaults_group_suffix}") + fi + if [ -n "${option_port}" ]; then + connect_options+=(--protocol=tcp) + connect_options+=(--port="${option_port}") + fi + if [ -n "${option_socket}" ]; then + connect_options+=(--protocol=socket) + connect_options+=(--socket="${option_socket}") + fi + if [ -n "${option_user}" ]; then + connect_options+=(--user="${option_user}") + fi + if [ -n "${option_password}" ]; then + connect_options+=(--password="${option_password}") + fi + + declare -a options + options=() + options+=(--sleep=0) + + dump_cmd="pt-mysql-summary ${options[*]} -- ${connect_options[*]}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} 2> "${error_file}" > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: pt-mysql-summary to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" + else + log "LOCAL_TASKS - ${FUNCNAME[0]}: 'pt-mysql-summary' not found, unable to dump summary" + fi +} + +####################################################################### +# Dump grants of an instance +# +# Arguments: +# --port=[Integer] (default: ) +# --socket=[String] (default: ) +# --user=[String] (default: ) +# --password=[String] (default: ) +# --defaults-file=[String] (default: ) +# --dump-label=[String] (default: "default") +# used as suffix of the dump dir to differenciate multiple instances +####################################################################### +dump_mysql_grants() { + local option_port="" + local option_socket="" + local option_defaults_file="" + local option_user="" + local option_password="" + local option_dump_label="" + + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case ${1:-''} in + --defaults-file) + # defaults-file options, with value separated by space + if [ -n "$2" ]; then + option_defaults_file="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-file=?*) + # defaults-file options, with value separated by = + option_defaults_file="${1#*=}" + ;; + --defaults-file=) + # defaults-file options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + ;; + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + ;; + --socket) + # socket options, with value separated by space + if [ -n "$2" ]; then + option_socket="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--socket' requires a non-empty option argument." + exit 1 + fi + ;; + --socket=?*) + # socket options, with value separated by = + option_socket="${1#*=}" + ;; + --socket=) + # socket options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--socket' requires a non-empty option argument." + exit 1 + ;; + --user) + # user options, with value separated by space + if [ -n "$2" ]; then + option_user="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + fi + ;; + --user=?*) + # user options, with value separated by = + option_user="${1#*=}" + ;; + --user=) + # user options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + ;; + --password) + # password options, with value separated by space + if [ -n "$2" ]; then + option_password="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + fi + ;; + --password=?*) + # password options, with value separated by = + option_password="${1#*=}" + ;; + --password=) + # password options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + ;; + --dump-label) + # dump-label options, with value separated by space + if [ -n "$2" ]; then + option_dump_label="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + fi + ;; + --dump-label=?*) + # dump-label options, with value separated by = + option_dump_label="${1#*=}" + ;; + --dump-label=) + # dump-label options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: unknown option '${1}' (ignored)" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + if [ -z "${option_dump_label}" ]; then + if [ -n "${option_port}" ]; then + option_dump_label="${option_port}" + elif [ -n "${option_socket}" ]; then + option_dump_label=$(path_to_str "${option_socket}") + else + option_dump_label="default" + fi + fi + + local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}-grants" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + ## Dump all grants (requires 'percona-toolkit' package) + if command -v pt-show-grants > /dev/null; then + local error_file="${errors_dir}/all_grants.err" + local dump_file="${dump_dir}/all_grants.sql" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + declare -a options + options=() + if [ -n "${option_defaults_file}" ]; then + options+=(--defaults-file="${option_defaults_file}") + fi + if [ -n "${option_port}" ]; then + options+=(--port="${option_port}") + fi + if [ -n "${option_socket}" ]; then + options+=(--socket="${option_socket}") + fi + if [ -n "${option_user}" ]; then + options+=(--user="${option_user}") + fi + if [ -n "${option_password}" ]; then + options+=(--password="${option_password}") + fi + options+=(--flush) + options+=(--no-header) + + dump_cmd="pt-show-grants ${options[*]}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} 2> "${error_file}" > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: pt-show-grants to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" + else + log "LOCAL_TASKS - ${FUNCNAME[0]}: 'pt-show-grants' not found, unable to dump grants" + fi +} + +####################################################################### +# Dump a single compressed file of all databases of an instance +# and a file containing only the schema. +# +# Arguments: +# --masterdata (default: ) +# --port=[Integer] (default: ) +# --socket=[String] (default: ) +# --user=[String] (default: ) +# --password=[String] (default: ) +# --defaults-file=[String] (default: ) +# --defaults-extra-file=[String] (default: ) +# --defaults-group-suffix=[String] (default: ) +# --dump-label=[String] (default: "default") +# used as suffix of the dump dir to differenciate multiple instances +# --compress= (default: "gzip") +# Other options after -- are passed as-is to mysqldump +####################################################################### +dump_mysql_global() { + local option_masterdata="" + local option_port="" + local option_socket="" + local option_defaults_file="" + local option_defaults_extra_file="" + local option_defaults_group_suffix="" + local option_user="" + local option_password="" + local option_dump_label="" + local option_compress="" + local option_others="" + + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case ${1:-''} in + --masterdata) + option_masterdata="--masterdata" + ;; + --defaults-file) + # defaults-file options, with value separated by space + if [ -n "$2" ]; then + option_defaults_file="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-file=?*) + # defaults-file options, with value separated by = + option_defaults_file="${1#*=}" + ;; + --defaults-file=) + # defaults-file options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + ;; + --defaults-extra-file) + # defaults-file options, with value separated by space + if [ -n "$2" ]; then + option_defaults_extra_file="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-extra-file=?*) + # defaults-extra-file options, with value separated by = + option_defaults_extra_file="${1#*=}" + ;; + --defaults-extra-file=) + # defaults-extra-file options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-extra-file' requires a non-empty option argument." + exit 1 + ;; + --defaults-group-suffix) + # defaults-group-suffix options, with value separated by space + if [ -n "$2" ]; then + option_defaults_group_suffix="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-group-suffix' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-group-suffix=?*) + # defaults-group-suffix options, with value separated by = + option_defaults_group_suffix="${1#*=}" + ;; + --defaults-group-suffix=) + # defaults-group-suffix options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-group-suffix' requires a non-empty option argument." + exit 1 + ;; + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + ;; + --socket) + # socket options, with value separated by space + if [ -n "$2" ]; then + option_socket="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--socket' requires a non-empty option argument." + exit 1 + fi + ;; + --socket=?*) + # socket options, with value separated by = + option_socket="${1#*=}" + ;; + --socket=) + # socket options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--socket' requires a non-empty option argument." + exit 1 + ;; + --user) + # user options, with value separated by space + if [ -n "$2" ]; then + option_user="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + fi + ;; + --user=?*) + # user options, with value separated by = + option_user="${1#*=}" + ;; + --user=) + # user options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + ;; + --password) + # password options, with value separated by space + if [ -n "$2" ]; then + option_password="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + fi + ;; + --password=?*) + # password options, with value separated by = + option_password="${1#*=}" + ;; + --password=) + # password options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + ;; + --dump-label) + # dump-label options, with value separated by space + if [ -n "$2" ]; then + option_dump_label="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + fi + ;; + --dump-label=?*) + # dump-label options, with value separated by = + option_dump_label="${1#*=}" + ;; + --dump-label=) + # dump-label options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + ;; + --compress) + # compress options, with value separated by space + if [ -n "$2" ]; then + option_compress="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument." + exit 1 + fi + ;; + --compress=?*) + # compress options, with value separated by = + option_compress="${1#*=}" + ;; + --compress=) + # compress options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + option_others=${*} + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: unknown option '${1}' (ignored)" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + case "${option_compress}" in + none) + compress_cmd="cat" + dump_ext="" + ;; + bzip2|bz|bz2) + compress_cmd="bzip2 --best" + dump_ext=".bz" + ;; + xz) + compress_cmd="xz --best" + dump_ext=".xz" + ;; + pigz) + compress_cmd="pigz --best" + dump_ext=".gz" + ;; + gz|gzip|*) + compress_cmd="gzip --best" + dump_ext=".gz" + ;; + esac + + if [ -z "${option_dump_label}" ]; then + if [ -n "${option_defaults_group_suffix}" ]; then + option_dump_label="${option_defaults_group_suffix}" + elif [ -n "${option_port}" ]; then + option_dump_label="${option_port}" + elif [ -n "${option_socket}" ]; then + option_dump_label=$(path_to_str "${option_socket}") + else + option_dump_label="default" + fi + fi + + ## Connection options + declare -a connect_options + connect_options=() + if [ -n "${option_defaults_file}" ]; then + connect_options+=(--defaults-file="${option_defaults_file}") + fi + if [ -n "${option_defaults_extra_file}" ]; then + connect_options+=(--defaults-extra-file="${option_defaults_extra_file}") + fi + if [ -n "${option_defaults_group_suffix}" ]; then + connect_options+=(--defaults-group-suffix="${option_defaults_group_suffix}") + fi + if [ -n "${option_port}" ]; then + connect_options+=(--protocol=tcp) + connect_options+=(--port="${option_port}") + fi + if [ -n "${option_socket}" ]; then + connect_options+=(--protocol=socket) + connect_options+=(--socket="${option_socket}") + fi + if [ -n "${option_user}" ]; then + connect_options+=(--user="${option_user}") + fi + if [ -n "${option_password}" ]; then + connect_options+=(--password="${option_password}") + fi + + ## Global all databases in one file + + local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + local error_file="${errors_dir}/mysqldump.err" + local dump_file="${dump_dir}/mysqldump.sql${dump_ext}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + declare -a dump_options + dump_options=() + dump_options+=(--opt) + dump_options+=(--force) + dump_options+=(--events) + dump_options+=(--hex-blob) + dump_options+=(--all-databases) + if [ -n "${option_masterdata}" ]; then + dump_options+=("${option_masterdata}") + fi + if [ -n "${option_others}" ]; then + # word splitting is deliberate here + # shellcheck disable=SC2206 + dump_options+=(${option_others}) + fi + + ## WARNING : logging and executing the command must be separate + ## because otherwise Bash would interpret | and > as strings and not syntax. + + dump_cmd="mysqldump ${connect_options[*]} ${dump_options[*]} 2> ${error_file} | ${compress_cmd} > ${dump_file}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + mysqldump "${connect_options[@]}" "${dump_options[@]}" 2> "${error_file}" | ${compress_cmd} > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" + + + ## Schema only (no data) for each databases + + local error_file="${errors_dir}/mysqldump.schema.err" + local dump_file="${dump_dir}/mysqldump.schema.sql" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + declare -a dump_options + dump_options=() + dump_options+=(--force) + dump_options+=(--no-data) + dump_options+=(--all-databases) + if [ -n "${option_others}" ]; then + # word splitting is deliberate here + # shellcheck disable=SC2206 + dump_options+=(${option_others}) + fi + + dump_cmd="mysqldump ${connect_options[*]} ${dump_options[*]}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} 2> "${error_file}" > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" +} + +####################################################################### +# Dump a file of each databases of an instance +# and a file containing only the schema. +# +# Arguments: +# --port=[Integer] (default: ) +# --socket=[String] (default: ) +# --user=[String] (default: ) +# --password=[String] (default: ) +# --defaults-file=[String] (default: ) +# --defaults-extra-file=[String] (default: ) +# --defaults-group-suffix=[String] (default: ) +# --dump-label=[String] (default: "default") +# used as suffix of the dump dir to differenciate multiple instances +# --compress= (default: "gzip") +# Other options after -- are passed as-is to mysqldump +####################################################################### +dump_mysql_per_base() { + local option_port="" + local option_socket="" + local option_defaults_file="" + local option_defaults_extra_file="" + local option_defaults_group_suffix="" + local option_user="" + local option_password="" + local option_dump_label="" + local option_compress="" + local option_others="" + + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case ${1:-''} in + --defaults-file) + # defaults-file options, with value separated by space + if [ -n "$2" ]; then + option_defaults_file="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-file=?*) + # defaults-file options, with value separated by = + option_defaults_file="${1#*=}" + ;; + --defaults-file=) + # defaults-file options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + ;; + --defaults-extra-file) + # defaults-file options, with value separated by space + if [ -n "$2" ]; then + option_defaults_extra_file="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-extra-file=?*) + # defaults-extra-file options, with value separated by = + option_defaults_extra_file="${1#*=}" + ;; + --defaults-extra-file=) + # defaults-extra-file options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-extra-file' requires a non-empty option argument." + exit 1 + ;; + --defaults-group-suffix) + # defaults-group-suffix options, with value separated by space + if [ -n "$2" ]; then + option_defaults_group_suffix="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-group-suffix' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-group-suffix=?*) + # defaults-group-suffix options, with value separated by = + option_defaults_group_suffix="${1#*=}" + ;; + --defaults-group-suffix=) + # defaults-group-suffix options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-group-suffix' requires a non-empty option argument." + exit 1 + ;; + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + ;; + --socket) + # socket options, with value separated by space + if [ -n "$2" ]; then + option_socket="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--socket' requires a non-empty option argument." + exit 1 + fi + ;; + --socket=?*) + # socket options, with value separated by = + option_socket="${1#*=}" + ;; + --socket=) + # socket options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--socket' requires a non-empty option argument." + exit 1 + ;; + --user) + # user options, with value separated by space + if [ -n "$2" ]; then + option_user="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + fi + ;; + --user=?*) + # user options, with value separated by = + option_user="${1#*=}" + ;; + --user=) + # user options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + ;; + --password) + # password options, with value separated by space + if [ -n "$2" ]; then + option_password="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + fi + ;; + --password=?*) + # password options, with value separated by = + option_password="${1#*=}" + ;; + --password=) + # password options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + ;; + --dump-label) + # dump-label options, with value separated by space + if [ -n "$2" ]; then + option_dump_label="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + fi + ;; + --dump-label=?*) + # dump-label options, with value separated by = + option_dump_label="${1#*=}" + ;; + --dump-label=) + # dump-label options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + ;; + --compress) + # compress options, with value separated by space + if [ -n "$2" ]; then + option_compress="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument." + exit 1 + fi + ;; + --compress=?*) + # compress options, with value separated by = + option_compress="${1#*=}" + ;; + --compress=) + # compress options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + option_others=${*} + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: unknown option '${1}' (ignored)" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + case "${option_compress}" in + none) + compress_cmd="cat" + dump_ext="" + ;; + bzip2|bz|bz2) + compress_cmd="bzip2 --best" + dump_ext=".bz" + ;; + xz) + compress_cmd="xz --best" + dump_ext=".xz" + ;; + pigz) + compress_cmd="pigz --best" + dump_ext=".gz" + ;; + gz|gzip|*) + compress_cmd="gzip --best" + dump_ext=".gz" + ;; + esac + + if [ -z "${option_dump_label}" ]; then + if [ -n "${option_defaults_group_suffix}" ]; then + option_dump_label="${option_defaults_group_suffix}" + elif [ -n "${option_port}" ]; then + option_dump_label="${option_port}" + elif [ -n "${option_socket}" ]; then + option_dump_label=$(path_to_str "${option_socket}") + else + option_dump_label="default" + fi + fi + + ## Connection options + declare -a connect_options + connect_options=() + if [ -n "${option_defaults_file}" ]; then + connect_options+=(--defaults-file="${option_defaults_file}") + fi + if [ -n "${option_defaults_extra_file}" ]; then + connect_options+=(--defaults-extra-file="${option_defaults_extra_file}") + fi + if [ -n "${option_defaults_group_suffix}" ]; then + connect_options+=(--defaults-group-suffix="${option_defaults_group_suffix}") + fi + if [ -n "${option_port}" ]; then + connect_options+=(--protocol=tcp) + connect_options+=(--port="${option_port}") + fi + if [ -n "${option_socket}" ]; then + connect_options+=(--protocol=socket) + connect_options+=(--socket="${option_socket}") + fi + if [ -n "${option_user}" ]; then + connect_options+=(--user="${option_user}") + fi + if [ -n "${option_password}" ]; then + connect_options+=(--password="${option_password}") + fi + + local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}-per-base" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + databases=$(mysql "${connect_options[@]}" --execute="show databases" --silent --skip-column-names \ + | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)") + + for database in ${databases}; do + local error_file="${errors_dir}/${database}.err" + local dump_file="${dump_dir}/${database}.sql${dump_ext}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + declare -a dump_options + dump_options=() + dump_options+=(--opt) + dump_options+=(--force) + dump_options+=(--events) + dump_options+=(--hex-blob) + dump_options+=(--databases "${database}") + if [ -n "${option_others}" ]; then + # word splitting is deliberate here + # shellcheck disable=SC2206 + dump_options+=(${option_others}) + fi + + ## WARNING : logging and executing the command must be separate + ## because otherwise Bash would interpret | and > as strings and not syntax. + + log "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump ${connect_options[*]} ${dump_options[*]} | ${compress_cmd} > ${dump_file}" + mysqldump "${connect_options[@]}" "${dump_options[@]}" 2> "${error_file}" | ${compress_cmd} > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" + + + ## Schema only (no data) for each databases + + local error_file="${errors_dir}/${database}.schema.err" + local dump_file="${dump_dir}/${database}.schema.sql" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + declare -a dump_options + dump_options=() + dump_options+=(--force) + dump_options+=(--no-data) + dump_options+=(--databases "${database}") + if [ -n "${option_others}" ]; then + # word splitting is deliberate here + # shellcheck disable=SC2206 + dump_options+=(${option_others}) + fi + + dump_cmd="mysqldump ${connect_options[*]} ${dump_options[*]}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} 2> "${error_file}" > "${dump_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" + done +} + +####################################################################### +# Dump "tabs style" separate schema/data for each database of an instance +# +# Arguments: +# --port=[Integer] (default: ) +# --socket=[String] (default: ) +# --user=[String] (default: ) +# --password=[String] (default: ) +# --defaults-file=[String] (default: ) +# --defaults-extra-file=[String] (default: ) +# --defaults-group-suffix=[String] (default: ) +# --dump-label=[String] (default: "default") +# used as suffix of the dump dir to differenciate multiple instances +# --compress= (default: "gzip") +# Other options after -- are passed as-is to mysqldump +####################################################################### +dump_mysql_tabs() { + local option_port="" + local option_socket="" + local option_defaults_file="" + local option_defaults_extra_file="" + local option_defaults_group_suffix="" + local option_user="" + local option_password="" + local option_dump_label="" + local option_compress="" + local option_others="" + + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case ${1:-''} in + --defaults-file) + # defaults-file options, with value separated by space + if [ -n "$2" ]; then + option_defaults_file="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-file=?*) + # defaults-file options, with value separated by = + option_defaults_file="${1#*=}" + ;; + --defaults-file=) + # defaults-file options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + ;; + --defaults-extra-file) + # defaults-file options, with value separated by space + if [ -n "$2" ]; then + option_defaults_extra_file="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-extra-file=?*) + # defaults-extra-file options, with value separated by = + option_defaults_extra_file="${1#*=}" + ;; + --defaults-extra-file=) + # defaults-extra-file options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-extra-file' requires a non-empty option argument." + exit 1 + ;; + --defaults-group-suffix) + # defaults-group-suffix options, with value separated by space + if [ -n "$2" ]; then + option_defaults_group_suffix="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-group-suffix' requires a non-empty option argument." + exit 1 + fi + ;; + --defaults-group-suffix=?*) + # defaults-group-suffix options, with value separated by = + option_defaults_group_suffix="${1#*=}" + ;; + --defaults-group-suffix=) + # defaults-group-suffix options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-group-suffix' requires a non-empty option argument." + exit 1 + ;; + --port) + # port options, with value separated by space + if [ -n "$2" ]; then + option_port="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + fi + ;; + --port=?*) + # port options, with value separated by = + option_port="${1#*=}" + ;; + --port=) + # port options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--port' requires a non-empty option argument." + exit 1 + ;; + --socket) + # socket options, with value separated by space + if [ -n "$2" ]; then + option_socket="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--socket' requires a non-empty option argument." + exit 1 + fi + ;; + --socket=?*) + # socket options, with value separated by = + option_socket="${1#*=}" + ;; + --socket=) + # socket options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--socket' requires a non-empty option argument." + exit 1 + ;; + --user) + # user options, with value separated by space + if [ -n "$2" ]; then + option_user="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + fi + ;; + --user=?*) + # user options, with value separated by = + option_user="${1#*=}" + ;; + --user=) + # user options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument." + exit 1 + ;; + --password) + # password options, with value separated by space + if [ -n "$2" ]; then + option_password="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + fi + ;; + --password=?*) + # password options, with value separated by = + option_password="${1#*=}" + ;; + --password=) + # password options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument." + exit 1 + ;; + --dump-label) + # dump-label options, with value separated by space + if [ -n "$2" ]; then + option_dump_label="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + fi + ;; + --dump-label=?*) + # dump-label options, with value separated by = + option_dump_label="${1#*=}" + ;; + --dump-label=) + # dump-label options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + ;; + --compress) + # compress options, with value separated by space + if [ -n "$2" ]; then + option_compress="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument." + exit 1 + fi + ;; + --compress=?*) + # compress options, with value separated by = + option_compress="${1#*=}" + ;; + --compress=) + # compress options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + option_others=${*} + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: unknown option '${1}' (ignored)" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + case "${option_compress}" in + none) + compress_cmd="cat" + dump_ext="" + ;; + bzip2|bz|bz2) + compress_cmd="bzip2 --best" + dump_ext=".bz" + ;; + xz) + compress_cmd="xz --best" + dump_ext=".xz" + ;; + pigz) + compress_cmd="pigz --best" + dump_ext=".gz" + ;; + gz|gzip|*) + compress_cmd="gzip --best" + dump_ext=".gz" + ;; + esac + + if [ -z "${option_dump_label}" ]; then + if [ -n "${option_defaults_group_suffix}" ]; then + option_dump_label="${option_defaults_group_suffix}" + elif [ -n "${option_port}" ]; then + option_dump_label="${option_port}" + elif [ -n "${option_socket}" ]; then + option_dump_label=$(path_to_str "${option_socket}") + else + option_dump_label="default" + fi + fi + + ## Connection options + declare -a connect_options + connect_options=() + if [ -n "${option_defaults_file}" ]; then + connect_options+=(--defaults-file="${option_defaults_file}") + fi + if [ -n "${option_defaults_extra_file}" ]; then + connect_options+=(--defaults-extra-file="${option_defaults_extra_file}") + fi + if [ -n "${option_defaults_group_suffix}" ]; then + connect_options+=(--defaults-group-suffix="${option_defaults_group_suffix}") + fi + if [ -n "${option_port}" ]; then + connect_options+=(--protocol=tcp) + connect_options+=(--port="${option_port}") + fi + if [ -n "${option_socket}" ]; then + connect_options+=(--protocol=socket) + connect_options+=(--socket="${option_socket}") + fi + if [ -n "${option_user}" ]; then + connect_options+=(--user="${option_user}") + fi + if [ -n "${option_password}" ]; then + connect_options+=(--password="${option_password}") + fi + + databases=$(mysql "${connect_options[@]}" --execute="show databases" --silent --skip-column-names \ + | grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)") + + for database in ${databases}; do + local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}-tabs/${database}" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + chown -RL mysql "${dump_dir}" + + local error_file="${errors_dir}.err" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_dir}" + + declare -a dump_options + dump_options=() + dump_options+=(--force) + dump_options+=(--quote-names) + dump_options+=(--opt) + dump_options+=(--events) + dump_options+=(--hex-blob) + dump_options+=(--skip-comments) + dump_options+=(--fields-enclosed-by='\"') + dump_options+=(--fields-terminated-by=',') + dump_options+=(--tab="${dump_dir}") + if [ -n "${option_others}" ]; then + # word splitting is deliberate here + # shellcheck disable=SC2206 + dump_options+=(${option_others}) + fi + dump_options+=("${database}") + + dump_cmd="mysqldump ${connect_options[*]} ${dump_options[*]}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} 2> "${error_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump to ${dump_dir} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_dir}" + done +} diff --git a/evobackup-client/files/upstream/lib/dump/postgresql.sh b/evobackup-client/files/upstream/lib/dump/postgresql.sh new file mode 100644 index 00000000..302942c1 --- /dev/null +++ b/evobackup-client/files/upstream/lib/dump/postgresql.sh @@ -0,0 +1,343 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2034,SC2317,SC2155 + +####################################################################### +# Dump a single file of all PostgreSQL databases +# +# Arguments: +# --dump-label=[String] (default: "default") +# used as suffix of the dump dir to differenciate multiple instances +# --compress= (default: "gzip") +# Other options after -- are passed as-is to pg_dump +####################################################################### +dump_postgresql_global() { + local option_dump_label="" + local option_compress="" + local option_others="" + + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case ${1:-''} in + --dump-label) + # dump-label options, with value separated by space + if [ -n "$2" ]; then + option_dump_label="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + fi + ;; + --dump-label=?*) + # dump-label options, with value separated by = + option_dump_label="${1#*=}" + ;; + --dump-label=) + # dump-label options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + ;; + --compress) + # compress options, with value separated by space + if [ -n "$2" ]; then + option_compress="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument." + exit 1 + fi + ;; + --compress=?*) + # compress options, with value separated by = + option_compress="${1#*=}" + ;; + --compress=) + # compress options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + option_others=${*} + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: unknown option '${1}' (ignored)" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + case "${option_compress}" in + none) + compress_cmd="cat" + dump_ext="" + ;; + bzip2|bz|bz2) + compress_cmd="bzip2 --best" + dump_ext=".bz" + ;; + xz) + compress_cmd="xz --best" + dump_ext=".xz" + ;; + pigz) + compress_cmd="pigz --best" + dump_ext=".gz" + ;; + gz|gzip|*) + compress_cmd="gzip --best" + dump_ext=".gz" + ;; + esac + + if [ -z "${option_dump_label}" ]; then + if [ -n "${option_defaults_group_suffix}" ]; then + option_dump_label="${option_defaults_group_suffix}" + elif [ -n "${option_port}" ]; then + option_dump_label="${option_port}" + elif [ -n "${option_socket}" ]; then + option_dump_label=$(path_to_str "${option_socket}") + else + option_dump_label="default" + fi + fi + + local dump_dir="${LOCAL_BACKUP_DIR}/postgresql-${option_dump_label}-global" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + ## example with pg_dumpall and with compression + local error_file="${errors_dir}/pg_dumpall.err" + local dump_file="${dump_dir}/pg_dumpall.sql${dump_ext}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + declare -a dump_options + dump_options=() + if [ -n "${option_others}" ]; then + # word splitting is deliberate here + # shellcheck disable=SC2206 + dump_options+=(${option_others}) + fi + + dump_cmd="(sudo -u postgres pg_dumpall ${dump_options[*]}) 2> ${error_file} | ${compress_cmd} > ${dump_file}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: pg_dumpall to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" + + ## example with pg_dumpall and without compression + ## WARNING: you need space in ~postgres + # local error_file="${errors_dir}/pg_dumpall.err" + # local dump_file="${dump_dir}/pg_dumpall.sql" + # log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + # + # (su - postgres -c "pg_dumpall > ~/pg.dump.bak") 2> "${error_file}" + # mv ~postgres/pg.dump.bak "${dump_file}" + # + # log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" +} + +####################################################################### +# Dump a compressed file per database +# +# Arguments: +####################################################################### +dump_postgresql_per_base() { + local option_dump_label="" + local option_compress="" + local option_others="" + + # Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a + while :; do + case ${1:-''} in + --dump-label) + # dump-label options, with value separated by space + if [ -n "$2" ]; then + option_dump_label="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + fi + ;; + --dump-label=?*) + # dump-label options, with value separated by = + option_dump_label="${1#*=}" + ;; + --dump-label=) + # dump-label options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument." + exit 1 + ;; + --compress) + # compress options, with value separated by space + if [ -n "$2" ]; then + option_compress="${2}" + shift + else + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument." + exit 1 + fi + ;; + --compress=?*) + # compress options, with value separated by = + option_compress="${1#*=}" + ;; + --compress=) + # compress options, without value + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument." + exit 1 + ;; + --) + # End of all options. + shift + option_others=${*} + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: unknown option '${1}' (ignored)" + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift + done + + case "${option_compress}" in + none) + compress_cmd="cat" + dump_ext="" + ;; + bzip2|bz|bz2) + compress_cmd="bzip2 --best" + dump_ext=".bz" + ;; + xz) + compress_cmd="xz --best" + dump_ext=".xz" + ;; + pigz) + compress_cmd="pigz --best" + dump_ext=".gz" + ;; + gz|gzip|*) + compress_cmd="gzip --best" + dump_ext=".gz" + ;; + esac + + if [ -z "${option_dump_label}" ]; then + if [ -n "${option_defaults_group_suffix}" ]; then + option_dump_label="${option_defaults_group_suffix}" + elif [ -n "${option_port}" ]; then + option_dump_label="${option_port}" + elif [ -n "${option_socket}" ]; then + option_dump_label=$(path_to_str "${option_socket}") + else + option_dump_label="default" + fi + fi + + local dump_dir="${LOCAL_BACKUP_DIR}/postgresql-${option_dump_label}-per-base" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + ( + # shellcheck disable=SC2164 + cd /var/lib/postgresql + databases=$(sudo -u postgres psql -U postgres -lt | awk -F \| '{print $1}' | grep -v "template.*") + for database in ${databases} ; do + local error_file="${errors_dir}/${database}.err" + local dump_file="${dump_dir}/${database}.sql${dump_ext}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + declare -a dump_options + dump_options=() + dump_options+=(--create) + dump_options+=(-U postgres) + dump_options+=(-d "${database}") + if [ -n "${option_others}" ]; then + # word splitting is deliberate here + # shellcheck disable=SC2206 + dump_options+=(${option_others}) + fi + + dump_cmd="(sudo -u postgres /usr/bin/pg_dump ${dump_options[*]}) 2> ${error_file} | ${compress_cmd} > ${dump_file}" + log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" + ${dump_cmd} + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: pg_dump to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" + done + ) +} + +####################################################################### +# Dump a compressed file per database +# +# Arguments: +# +# TODO: add arguments to include/exclude tables +####################################################################### +dump_postgresql_filtered() { + local dump_dir="${LOCAL_BACKUP_DIR}/postgresql-filtered" + local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") + rm -rf "${dump_dir}" "${errors_dir}" + mkdir -p "${dump_dir}" "${errors_dir}" + # No need to change recursively, the top directory is enough + chmod 700 "${dump_dir}" "${errors_dir}" + + local error_file="${errors_dir}/pg-backup.err" + local dump_file="${dump_dir}/pg-backup.tar" + log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" + + ## example with all tables from MYBASE excepts TABLE1 and TABLE2 + # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f "${dump_file}" -t 'TABLE1' -t 'TABLE2' MYBASE 2> "${error_file}" + + ## example with only TABLE1 and TABLE2 from MYBASE + # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f "${dump_file}" -T 'TABLE1' -T 'TABLE2' MYBASE 2> "${error_file}" + + local last_rc=$? + # shellcheck disable=SC2086 + if [ ${last_rc} -ne 0 ]; then + log_error "LOCAL_TASKS - ${FUNCNAME[0]}: pg_dump to ${dump_file} returned an error ${last_rc}" "${error_file}" + GLOBAL_RC=${E_DUMPFAILED} + else + rm -f "${error_file}" + fi + log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" +} diff --git a/evobackup-client/files/upstream/lib/main.sh b/evobackup-client/files/upstream/lib/main.sh new file mode 100644 index 00000000..d4873cb9 --- /dev/null +++ b/evobackup-client/files/upstream/lib/main.sh @@ -0,0 +1,466 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2034,SC2317 + +readonly VERSION="24.04.1" + +# set all programs to C language (english) +export LC_ALL=C + +# If expansion is attempted on an unset variable or parameter, the shell prints an +# error message, and, if not interactive, exits with a non-zero status. +set -o nounset +# The pipeline's return status is the value of the last (rightmost) command +# to exit with a non-zero status, or zero if all commands exit successfully. +set -o pipefail +# Enable trace mode if called with environment variable TRACE=1 +if [[ "${TRACE-0}" == "1" ]]; then + set -o xtrace +fi + +source "${LIBDIR}/utilities.sh" +source "${LIBDIR}/dump/elasticsearch.sh" +source "${LIBDIR}/dump/mysql.sh" +source "${LIBDIR}/dump/postgresql.sh" +source "${LIBDIR}/dump/misc.sh" + +# Called from main, it is wrapping the local_tasks function defined in the real script +local_tasks_wrapper() { + log "START LOCAL_TASKS" + + # Remove old log directories (recursively) + find "${LOCAL_BACKUP_DIR}/" -type d -name "${PROGNAME}.errors-*" -ctime +30 -exec rm -rf \; + + local_tasks_type="$(type -t local_tasks)" + if [ "${local_tasks_type}" = "function" ]; then + local_tasks + else + log_error "There is no 'local_tasks' function to execute" + fi + + # TODO: check if this is still needed + # print_error_files_content + + log "STOP LOCAL_TASKS" +} + +# Called from main, it is wrapping the sync_tasks function defined in the real script +sync_tasks_wrapper() { + declare -a SERVERS # Indexed array for server/port values + declare -a RSYNC_INCLUDES # Indexed array for includes + declare -a RSYNC_EXCLUDES # Indexed array for excludes + + case "${SYSTEM}" in + linux) + # NOTE: remember to single-quote paths if they contain globs (*) + # and you want to defer expansion + declare -a rsync_default_includes=( + /bin + /boot + /lib + /opt + /sbin + /usr + ) + ;; + *bsd) + # NOTE: remember to single-quote paths if they contain globs (*) + # and you want to defer expansion + declare -a rsync_default_includes=( + /bin + /bsd + /sbin + /usr + ) + ;; + *) + echo "Unknown system '${SYSTEM}'" >&2 + exit 1 + ;; + esac + if [ -f "${CANARY_FILE}" ]; then + rsync_default_includes+=("${CANARY_FILE}") + fi + readonly rsync_default_includes + + # NOTE: remember to single-quote paths if they contain globs (*) + # and you want to defer expansion + declare -a rsync_default_excludes=( + /dev + /proc + /run + /sys + /tmp + /usr/doc + /usr/obj + /usr/share/doc + /usr/src + /var/apt + /var/cache + '/var/db/munin/*.tmp' + /var/lib/amavis/amavisd.sock + /var/lib/amavis/tmp + /var/lib/amavis/virusmails + '/var/lib/clamav/*.tmp' + /var/lib/elasticsearch + /var/lib/metche + /var/lib/mongodb + '/var/lib/munin/*tmp*' + /var/lib/mysql + /var/lib/php/sessions + /var/lib/php5 + /var/lib/postgres + /var/lib/postgresql + /var/lib/sympa + /var/lock + /var/run + /var/spool/postfix + /var/spool/smtpd + /var/spool/squid + /var/state + /var/tmp + lost+found + '.nfs.*' + 'lxc/*/rootfs/tmp' + 'lxc/*/rootfs/usr/doc' + 'lxc/*/rootfs/usr/obj' + 'lxc/*/rootfs/usr/share/doc' + 'lxc/*/rootfs/usr/src' + 'lxc/*/rootfs/var/apt' + 'lxc/*/rootfs/var/cache' + 'lxc/*/rootfs/var/lib/php5' + 'lxc/*/rootfs/var/lib/php/sessions' + 'lxc/*/rootfs/var/lock' + 'lxc/*/rootfs/var/run' + 'lxc/*/rootfs/var/state' + 'lxc/*/rootfs/var/tmp' + /home/mysqltmp + ) + readonly rsync_default_excludes + + sync_tasks_type="$(type -t sync_tasks)" + if [ "${sync_tasks_type}" = "function" ]; then + sync_tasks + else + log_error "There is no 'sync_tasks' function to execute" + fi +} + +sync() { + local sync_name=${1} + local -a rsync_servers=("${!2}") + local -a rsync_includes=("${!3}") + local -a rsync_excludes=("${!4}") + + ## Initialize variable to store SSH connection errors + declare -a SSH_ERRORS=() + + log "START SYNC_TASKS - sync=${sync_name}" + + # echo "### sync ###" + + # for server in "${rsync_servers[@]}"; do + # echo "server: ${server}" + # done + + # for include in "${rsync_includes[@]}"; do + # echo "include: ${include}" + # done + + # for exclude in "${rsync_excludes[@]}"; do + # echo "exclude: ${exclude}" + # done + + local -i n=0 + local server="" + if [ "${SERVERS_FALLBACK}" = "1" ]; then + # We try to find a suitable server + while :; do + server=$(pick_server ${n} "${sync_name}") + rc=$? + if [ ${rc} != 0 ]; then + GLOBAL_RC=${E_NOSRVAVAIL} + log "STOP SYNC_TASKS - sync=${sync_name}'" + return + fi + + if test_server "${server}"; then + break + else + server="" + n=$(( n + 1 )) + fi + done + else + # we force the server + server=$(pick_server "${n}" "${sync_name}") + fi + + rsync_server=$(echo "${server}" | cut -d':' -f1) + rsync_port=$(echo "${server}" | cut -d':' -f2) + + log "SYNC_TASKS - sync=${sync_name}: use ${server}" + + # Rsync complete log file for the current run + RSYNC_LOGFILE="/var/log/${PROGNAME}.${sync_name}.rsync.log" + # Rsync stats for the current run + RSYNC_STATSFILE="/var/log/${PROGNAME}.${sync_name}.rsync-stats.log" + + # reset Rsync log file + if [ -n "$(command -v truncate)" ]; then + truncate -s 0 "${RSYNC_LOGFILE}" + truncate -s 0 "${RSYNC_STATSFILE}" + else + printf "" > "${RSYNC_LOGFILE}" + printf "" > "${RSYNC_STATSFILE}" + fi + + # Initialize variable here, we need it later + local -a mtree_files=() + + if [ "${MTREE_ENABLED}" = "1" ]; then + mtree_bin=$(command -v mtree) + + if [ -n "${mtree_bin}" ]; then + # Dump filesystem stats with mtree + log "SYNC_TASKS - sync=${sync_name}: start mtree" + + # Loop over Rsync includes + for i in "${!rsync_includes[@]}"; do + include="${rsync_includes[i]}" + + if [ -d "${include}" ]; then + # … but exclude for mtree what will be excluded by Rsync + mtree_excludes_file="$(mktemp --tmpdir "${PROGNAME}.${sync_name}.mtree-excludes.XXXXXX")" + add_to_temp_files "${mtree_excludes_file}" + + for j in "${!rsync_excludes[@]}"; do + echo "${rsync_excludes[j]}" | grep -E "^([^/]|${include})" | sed -e "s|^${include}|.|" >> "${mtree_excludes_file}" + done + + mtree_file="/var/log/evobackup.$(basename "${include}").mtree" + add_to_temp_files "${mtree_file}" + + ${mtree_bin} -x -c -p "${include}" -X "${mtree_excludes_file}" > "${mtree_file}" + mtree_files+=("${mtree_file}") + fi + done + + if [ "${#mtree_files[@]}" -le 0 ]; then + log_error "SYNC_TASKS - ${sync_name}: ERROR: mtree didn't produce any file" + fi + + log "SYNC_TASKS - sync=${sync_name}: stop mtree (files: ${mtree_files[*]})" + else + log "SYNC_TASKS - sync=${sync_name}: skip mtree (missing)" + fi + else + log "SYNC_TASKS - sync=${sync_name}: skip mtree (disabled)" + fi + + rsync_bin=$(command -v rsync) + # Build the final Rsync command + + # Rsync main options + rsync_main_args=() + rsync_main_args+=(--archive) + rsync_main_args+=(--itemize-changes) + rsync_main_args+=(--quiet) + rsync_main_args+=(--stats) + rsync_main_args+=(--human-readable) + rsync_main_args+=(--relative) + rsync_main_args+=(--partial) + rsync_main_args+=(--delete) + rsync_main_args+=(--delete-excluded) + rsync_main_args+=(--force) + rsync_main_args+=(--ignore-errors) + rsync_main_args+=(--log-file "${RSYNC_LOGFILE}") + rsync_main_args+=(--rsh "ssh -p ${rsync_port} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'") + + # Rsync excludes + for i in "${!rsync_excludes[@]}"; do + rsync_main_args+=(--exclude "${rsync_excludes[i]}") + done + + # Rsync local sources + rsync_main_args+=("${rsync_includes[@]}") + + # Rsync remote destination + rsync_main_args+=("root@${rsync_server}:${REMOTE_BACKUP_DIR}/") + + # … log it + log "SYNC_TASKS - sync=${sync_name}: Rsync main command : ${rsync_bin} ${rsync_main_args[*]}" + + # … execute it + ${rsync_bin} "${rsync_main_args[@]}" + + rsync_main_rc=$? + + # Copy last lines of rsync log to the main log + tail -n 30 "${RSYNC_LOGFILE}" >> "${LOGFILE}" + # Copy Rsync stats to special file + tail -n 30 "${RSYNC_LOGFILE}" | grep --invert-match --extended-regexp " [\<\>ch\.\*]\S{10} " > "${RSYNC_STATSFILE}" + + # We ignore rc=24 (vanished files) + if [ ${rsync_main_rc} -ne 0 ] && [ ${rsync_main_rc} -ne 24 ]; then + log_error "SYNC_TASKS - sync=${sync_name}: Rsync main command returned an error ${rsync_main_rc}" "${LOGFILE}" + GLOBAL_RC=${E_SYNCFAILED} + else + # Build the report Rsync command + local -a rsync_report_args + + rsync_report_args=() + + # Rsync options + rsync_report_args+=(--rsh "ssh -p ${rsync_port} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'") + + # Rsync local sources + if [ "${#mtree_files[@]}" -gt 0 ]; then + # send mtree files if there is any + rsync_report_args+=("${mtree_files[@]}") + fi + if [ -f "${RSYNC_LOGFILE}" ]; then + # send rsync full log file if it exists + rsync_report_args+=("${RSYNC_LOGFILE}") + fi + if [ -f "${RSYNC_STATSFILE}" ]; then + # send rsync stats log file if it exists + rsync_report_args+=("${RSYNC_STATSFILE}") + fi + + # Rsync remote destination + rsync_report_args+=("root@${rsync_server}:${REMOTE_LOG_DIR}/") + + # … log it + log "SYNC_TASKS - sync=${sync_name}: Rsync report command : ${rsync_bin} ${rsync_report_args[*]}" + + # … execute it + ${rsync_bin} "${rsync_report_args[@]}" + fi + + log "STOP SYNC_TASKS - sync=${sync_name}" +} + +setup() { + # Default return-code (0 == succes) + GLOBAL_RC=0 + + # Possible error codes + readonly E_NOSRVAVAIL=21 # No server is available + readonly E_SYNCFAILED=20 # Failed sync task + readonly E_DUMPFAILED=10 # Failed dump task + + # explicit PATH + PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin + + # System name (linux, openbsd…) + : "${SYSTEM:=$(uname | tr '[:upper:]' '[:lower:]')}" + + # Hostname (for logs and notifications) + : "${HOSTNAME:=$(hostname)}" + + # Store pid in a file named after this program's name + : "${PROGNAME:=$(basename "$0")}" + : "${PIDFILE:="/var/run/${PROGNAME}.pid"}" + + # Customize the log path if you want multiple scripts to have separate log files + : "${LOGFILE:="/var/log/evobackup.log"}" + + # Canary file to update before executing tasks + : "${CANARY_FILE:="/zzz_evobackup_canary"}" + + # Date format for log messages + : "${DATE_FORMAT:="%Y-%m-%d %H:%M:%S"}" + + # Should we fallback on other servers when the first one is unreachable? + : "${SERVERS_FALLBACK:=1}" + # timeout (in seconds) for SSH connections + : "${SSH_CONNECT_TIMEOUT:=90}" + + : "${LOCAL_BACKUP_DIR:="/home/backup"}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${LOCAL_BACKUP_DIR}" + + : "${ERRORS_DIR:="${LOCAL_BACKUP_DIR}/${PROGNAME}.errors-${START_TIME}"}" + # shellcheck disable=SC2174 + mkdir -p -m 700 "${ERRORS_DIR}" + + # Backup directory on remote server + : "${REMOTE_BACKUP_DIR:="/var/backup"}" + # Log directory in remote server + : "${REMOTE_LOG_DIR:="/var/log"}" + + # Email address for notifications + : "${MAIL:="root"}" + + # Email subject for notifications + : "${MAIL_SUBJECT:="[info] EvoBackup - Client ${HOSTNAME}"}" + + # Enable/disable local tasks (default: enabled) + : "${LOCAL_TASKS:=1}" + # Enable/disable sync tasks (default: enabled) + : "${SYNC_TASKS:=1}" + + # Enable/disable mtree (default: enabled) + : "${MTREE_ENABLED:=1}" + + # If "setup_custom" exists and is a function, let's call it + setup_custom_type="$(type -t setup_custom)" + if [ "${setup_custom_type}" = "function" ]; then + setup_custom + fi + + ## Force umask + umask 077 + + # Initialize a list of temporary files + declare -a TEMP_FILES=() + # Any file in this list will be deleted when the program exits + trap "cleanup" EXIT +} + + +run_evobackup() { + # Start timer + START_EPOCH=$(/bin/date +%s) + START_TIME=$(/bin/date +"%Y%m%d%H%M%S") + + # Configure variables and environment + setup + + log "START GLOBAL - VERSION=${VERSION} LOCAL_TASKS=${LOCAL_TASKS} SYNC_TASKS=${SYNC_TASKS}" + + # /!\ Only one backup processus can run at the sametime /!\ + # Based on PID file, kill any running process before continuing + enforce_single_process "${PIDFILE}" + + # Update canary to keep track of each run + update-evobackup-canary --who "${PROGNAME}" --file "${CANARY_FILE}" + + if [ "${LOCAL_TASKS}" = "1" ]; then + local_tasks_wrapper + fi + + if [ "${SYNC_TASKS}" = "1" ]; then + sync_tasks_wrapper + fi + + STOP_EPOCH=$(/bin/date +%s) + + case "${SYSTEM}" in + *bsd) + start_time=$(/bin/date -f "%s" -j "${START_EPOCH}" +"${DATE_FORMAT}") + stop_time=$(/bin/date -f "%s" -j "${STOP_EPOCH}" +"${DATE_FORMAT}") + ;; + *) + start_time=$(/bin/date --date="@${START_EPOCH}" +"${DATE_FORMAT}") + stop_time=$(/bin/date --date="@${STOP_EPOCH}" +"${DATE_FORMAT}") + ;; + esac + duration=$(( STOP_EPOCH - START_EPOCH )) + + log "STOP GLOBAL - start='${start_time}' stop='${stop_time}' duration=${duration}s" + + send_mail + + exit ${GLOBAL_RC} +} diff --git a/evobackup-client/files/upstream/lib/utilities.sh b/evobackup-client/files/upstream/lib/utilities.sh new file mode 100644 index 00000000..e14a4341 --- /dev/null +++ b/evobackup-client/files/upstream/lib/utilities.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash + +# Output a message to the log file +log() { + local msg="${1:-$(cat /dev/stdin)}" + local pid=$$ + + printf "[%s] %s[%s]: %s\\n" \ + "$(/bin/date +"${DATE_FORMAT}")" "${PROGNAME}" "${pid}" "${msg}" \ + >> "${LOGFILE}" +} +log_error() { + local error_msg=${1} + local error_file=${2:-""} + + if [ -n "${error_file}" ] && [ -f "${error_file}" ]; then + printf "\n### %s\n" "${error_msg}" >&2 + # shellcheck disable=SC2046 + if [ $(wc -l "${error_file}" | cut -d " " -f 1) -gt 30 ]; then + printf "~~~{%s (tail -30)}\n" "${error_file}" >&2 + tail -n 30 "${error_file}" >&2 + else + printf "~~~{%s}\n" "${error_file}" >&2 + cat "${error_file}" >&2 + fi + printf "~~~\n" >&2 + + log "${error_msg}, check ${error_file}" + else + printf "\n### %s\n" "${error_msg}" >&2 + + log "${error_msg}" + fi + +} +add_to_temp_files() { + TEMP_FILES+=("${1}") +} +# Remove all temporary file created during the execution +cleanup() { + # shellcheck disable=SC2086 + rm -f "${TEMP_FILES[@]}" + find "${ERRORS_DIR}" -type d -empty -delete +} +enforce_single_process() { + local pidfile=$1 + + if [ -e "${pidfile}" ]; then + pid=$(cat "${pidfile}") + # Does process still exist? + if kill -0 "${pid}" 2> /dev/null; then + # Killing the childs of evobackup. + for ppid in $(pgrep -P "${pid}"); do + kill -9 "${ppid}"; + done + # Then kill the main PID. + kill -9 "${pid}" + printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\\n" >&2 + else + rm -f "${pidfile}" + fi + fi + add_to_temp_files "${pidfile}" + + echo "$$" > "${pidfile}" +} + +# Build the error directory (inside ERRORS_DIR) based on the dump directory path +errors_dir_from_dump_dir() { + local dump_dir=$1 + local relative_path=$(realpath --relative-to="${LOCAL_BACKUP_DIR}" "${dump_dir}") + + # return absolute path + realpath --canonicalize-missing "${ERRORS_DIR}/${relative_path}" +} + +# Call test_server with "HOST:PORT" string +# It will return with 0 if the server is reachable. +# It will return with 1 and a message on stderr if not. +test_server() { + local item=$1 + # split HOST and PORT from the input string + local host=$(echo "${item}" | cut -d':' -f1) + local port=$(echo "${item}" | cut -d':' -f2) + + local new_error + + # Test if the server is accepting connections + ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" "${host}" -p "${port}" -t "exit" + # shellcheck disable=SC2181 + if [ $? = 0 ]; then + # SSH connection is OK + return 0 + else + # SSH connection failed + new_error=$(printf "Failed to connect to \`%s' within %s seconds" "${item}" "${SSH_CONNECT_TIMEOUT}") + log "${new_error}" + SSH_ERRORS+=("${new_error}") + + return 1 + fi +} + +# Call pick_server with an optional positive integer to get the nth server in the list. +pick_server() { + local -i increment=${1:-0} + local -i list_length=${#SERVERS[@]} + local sync_name=${2:""} + + if (( increment >= list_length )); then + # We've reached the end of the list + new_error="No more server available" + new_error="${new_error} for sync '${sync_name}'" + log "${new_error}" + SSH_ERRORS+=("${new_error}") + + # Log errors to stderr + for i in "${!SSH_ERRORS[@]}"; do + printf "%s\n" "${SSH_ERRORS[i]}" >&2 + done + + return 1 + fi + + # Extract the day of month, without leading 0 (which would give an octal based number) + today=$(/bin/date +%e) + # A salt is useful to randomize the starting point in the list + # but stay identical each time it's called for a server (based on hostname). + salt=$(hostname | cksum | cut -d' ' -f1) + # Pick an integer between 0 and the length of the SERVERS list + # It changes each day + n=$(( (today + salt + increment) % list_length )) + + echo "${SERVERS[n]}" +} + +send_mail() { + tail -20 "${LOGFILE}" | mail -s "${MAIL_SUBJECT}" "${MAIL}" +} + +path_to_str() { + echo "${1}" | sed -e 's|^/||; s|/$||; s|/|:|g' +} diff --git a/evobackup-client/files/upstream/lib/zzz_evobackup.sh b/evobackup-client/files/upstream/lib/zzz_evobackup.sh new file mode 100644 index 00000000..a8fcb314 --- /dev/null +++ b/evobackup-client/files/upstream/lib/zzz_evobackup.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash +# +# Evobackup client +# See https://gitea.evolix.org/evolix/evobackup +# +# This is a generated backup script made by: +# command: @COMMAND@ +# version: @VERSION@ +# date: @DATE@ + +####################################################################### +# +# You must configure the MAIL variable to receive notifications. +# +# There is some optional configuration that you can do +# at the end of this script. +# +####################################################################### + +# Email adress for notifications +MAIL=__NOTIFICATION_MAIL__ + +####################################################################### +# +# The "sync_tasks" function will be called by the "run_evobackup" function. +# +# You can customize the variables: +# * "SYNC_NAME" (String) +# * "SERVERS" (Array of HOST:PORT) +# * "RSYNC_INCLUDES" (Array of paths to include) +# * "RSYNC_EXCLUDES" (Array of paths to exclude) +# +# WARNING: remember to single-quote paths if they contain globs (*) +# and you want to pass them as-is to Rsync. +# +# The "sync" function can be called multiple times +# with a different set of variables. +# That way you can to sync to various destinations. +# +# Default includes/excludes are defined in the "main" library, +# referenced at this end of this file. +# +####################################################################### + +# shellcheck disable=SC2034 +sync_tasks() { + + ########## System-only backup (to Evolix servers) ################# + + SYNC_NAME="evolix-system" + SERVERS=( + __SRV0_HOST__:__SRV0_PORT__ + __SRV1_HOST__:__SRV1_PORT__ + ) + RSYNC_INCLUDES=( + "${rsync_default_includes[@]}" + /etc + /root + /var + ) + RSYNC_EXCLUDES=( + "${rsync_default_excludes[@]}" + ) + sync "${SYNC_NAME}" "SERVERS[@]" "RSYNC_INCLUDES[@]" "RSYNC_EXCLUDES[@]" + + + ########## Full backup (to client servers) ######################## + + ### SYNC_NAME="client-full" + ### SERVERS=( + ### client-backup00.evolix.net:2221 + ### client-backup01.evolix.net:2221 + ### ) + ### RSYNC_INCLUDES=( + ### "${rsync_default_includes[@]}" + ### /etc + ### /root + ### /var + ### /home + ### /srv + ### ) + ### RSYNC_EXCLUDES=( + ### "${rsync_default_excludes[@]}" + ### ) + ### sync "${SYNC_NAME}" "SERVERS[@]" "RSYNC_INCLUDES[@]" "RSYNC_EXCLUDES[@]" + +} + +####################################################################### +# +# The "local_tasks" function will be called by the "run_evobackup" function. +# +# You can call any available "dump_xxx" function +# (usually installed at /usr/local/lib/evobackup/dump-*.sh) +# +# You can also write some custom functions and call them. +# A "dump_custom" example is available further down. +# +####################################################################### + +local_tasks() { + + ########## Server state ########### + + # Run dump-server-state to extract system information + # + # Options : any dump-server-state supported option + # (except --dump-dir that will be overwritten) + # See 'dump-server-state -h' for details. + # + dump_server_state + + ########## MySQL ################## + + # Very common strategy for a single instance server with default configuration : + # + ### dump_mysql_global; dump_mysql_grants; dump_mysql_summary + # + # See below for details regarding dump functions for MySQL/MariaDB + + # Dump all databases in a single compressed file + # + # Options : + # --masterdata (default: ) + # --port=[Integer] (default: ) + # --socket=[String] (default: ) + # --user=[String] (default: ) + # --password=[String] (default: ) + # --defaults-file=[String] (default: ) + # --defaults-extra-file=[String] (default: ) + # --defaults-group-suffix=[String] (default: ) + # --dump-label=[String] (default: "default") + # used as suffix of the dump dir to differenciate multiple instances + # + ### dump_mysql_global + + # Dump each database separately, in a compressed file + # + # Options : + # --port=[Integer] (default: ) + # --socket=[String] (default: ) + # --user=[String] (default: ) + # --password=[String] (default: ) + # --defaults-file=[String] (default: ) + # --defaults-extra-file=[String] (default: ) + # --defaults-group-suffix=[String] (default: ) + # --dump-label=[String] (default: "default") + # used as suffix of the dump dir to differenciate multiple instances + # + ### dump_mysql_per_base + + # Dump permissions of an instance (using pt-show-grants) + # + # Options : + # --port=[Integer] (default: ) + # --socket=[String] (default: ) + # --user=[String] (default: ) + # --password=[String] (default: ) + # --defaults-file=[String] (default: ) + # --dump-label=[String] (default: "default") + # used as suffix of the dump dir to differenciate multiple instances + # + # WARNING - unsupported options : + # --defaults-extra-file + # --defaults-group-suffix + # You have to provide credentials manually + # + ### dump_mysql_grants + + # Dump complete summary of an instance (using pt-mysql-summary) + # + # Options : + # --port=[Integer] (default: ) + # --socket=[String] (default: ) + # --user=[String] (default: ) + # --password=[String] (default: ) + # --defaults-file=[String] (default: ) + # --defaults-extra-file=[String] (default: ) + # --defaults-group-suffix=[String] (default: ) + # --dump-label=[String] (default: "default") + # used as suffix of the dump dir to differenciate multiple instances + # + ### dump_mysql_summary + + # Dump each table in separate schema/data files + # + # Options : + # --port=[Integer] (default: ) + # --socket=[String] (default: ) + # --user=[String] (default: ) + # --password=[String] (default: ) + # --defaults-file=[String] (default: ) + # --defaults-extra-file=[String] (default: ) + # --defaults-group-suffix=[String] (default: ) + # --dump-label=[String] (default: "default") + # used as suffix of the dump dir to differenciate multiple instances + # + ### dump_mysql_tabs + + ########## PostgreSQL ############# + + # Dump all databases in a single file (compressed or not) + # + ### dump_postgresql_global + + # Dump a specific databse with only some tables, or all but some tables (must be configured) + # + ### dump_postgresql_filtered + + # Dump each database separately, in a compressed file + # + ### dump_postgresql_per_base + + ########## MongoDB ################ + + ### dump_mongodb [--user=foo] [--password=123456789] + + ########## Redis ################## + + # Copy data file for all instances + # + ### dump_redis [--instances=] + + ########## Elasticsearch ########## + + # Snapshot data for a single-node cluster + # + ### dump_elasticsearch_snapshot_singlenode [--protocol=http] [--host=localhost] [--port=9200] [--user=foo] [--password=123456789] [--repository=snaprepo] [--snapshot=snapshot.daily] + + # Snapshot data for a multi-node cluster + # + ### dump_elasticsearch_snapshot_multinode [--protocol=http] [--host=localhost] [--port=9200] [--user=foo] [--password=123456789] [--repository=snaprepo] [--snapshot=snapshot.daily] [--nfs-server=192.168.2.1] + + ########## RabbitMQ ############### + + ### dump_rabbitmq + + ########## MegaCli ################ + + # Copy RAID config + # + ### dump_megacli_config + + # Dump file access control lists + # + ### dump_facl + + ########## OpenLDAP ############### + + ### dump_ldap + + ########## Network ################ + + # Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) + # + ### dump_traceroute --targets=host_or_ip[,host_or_ip] + dump_traceroute --targets=8.8.8.8,www.evolix.fr,travaux.evolix.net + + # No-op, in case nothing is enabled + : +} + +# This is an example for a custom dump function +# Uncomment, customize and call it from the "local_tasks" function +### dump_custom() { +### # Set dump and errors directories and files +### local dump_dir="${LOCAL_BACKUP_DIR}/custom" +### local dump_file="${dump_dir}/dump.gz" +### local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}") +### local error_file="${errors_dir}/dump.err" +### +### # Reset dump and errors directories +### rm -rf "${dump_dir}" "${errors_dir}" +### # shellcheck disable=SC2174 +### mkdir -p -m 700 "${dump_dir}" "${errors_dir}" +### +### # Log the start of the function +### log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}" +### +### # Prepare the dump command (errors go to the error file and the data to the dump file) +### dump_cmd="my-dump-command 2> ${error_file} > ${dump_file}" +### log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd}" +### +### # Execute the dump command +### ${dump_cmd} +### +### # Check result and deal with potential errors +### local last_rc=$? +### # shellcheck disable=SC2086 +### if [ ${last_rc} -ne 0 ]; then +### log_error "LOCAL_TASKS - ${FUNCNAME[0]}: my-dump-command to ${dump_file} returned an error ${last_rc}" "${error_file}" +### GLOBAL_RC=${E_DUMPFAILED} +### else +### rm -f "${error_file}" +### fi +### +### # Log the end of the function +### log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}" +### } + +########## Optional configuration ##################################### + +setup_custom() { + # System name ("linux" and "openbsd" currently supported) + ### SYSTEM="$(uname)" + + # Host name for logs and notifications + ### HOSTNAME="$(hostname)" + + # Email subject for notifications + ### MAIL_SUBJECT="[info] EvoBackup - Client ${HOSTNAME}" + + # No-op in case nothing is executed + : +} + +########## Libraries ################################################## + +# Change this to wherever you install the libraries +LIBDIR="/usr/local/lib/evobackup" + +source "${LIBDIR}/main.sh" + +########## Let's go! ################################################## + +run_evobackup diff --git a/evobackup-client/tasks/install.yml b/evobackup-client/tasks/install.yml new file mode 100644 index 00000000..0e428331 --- /dev/null +++ b/evobackup-client/tasks/install.yml @@ -0,0 +1,50 @@ +--- + +- name: Dependencies are present + ansible.builtin.apt: + name: + - rsync + - mtree-netbsd + state: present + +- name: "Remount /usr if needed" + include_role: + name: remount-usr + when: evobackup_client__lib_dir is search("/usr") or evobackup_client__bin_dir is search("/usr") + +- name: copy evobackup libs + ansible.builtin.copy: + src: upstream/lib + dest: "{{ evobackup_client__lib_dir }}/" + force: True + mode: "0644" + owner: root + group: root + +- name: copy evobackupctl script + ansible.builtin.copy: + src: upstream/bin/evobackupctl + dest: "{{ evobackup_client__bin_dir }}/evobackupctl" + force: True + mode: "0755" + owner: root + group: root + +- name: LIBDIR is customized in evobackupctl + ansible.builtin.replace: + path: "{{ evobackup_client__bin_dir }}/evobackupctl" + regexp: "^LIBDIR=.+" + replace: "LIBDIR=\"{{ evobackup_client__lib_dir }}\"" + +- name: Evobackup canary cron is present + ansible.builtin.template: + src: update-evobackup-canary.sh.j2 + dest: "{{ evobackup_client__update_canary_path }}" + mode: "0700" + when: evobackup_client__update_canary_enable | bool + +- name: Evobackup canary cron is absent + ansible.builtin.file: + path: "{{ evobackup_client__update_canary_path }}" + state: absent + when: not ( evobackup_client__update_canary_enable | bool) diff --git a/evobackup-client/tasks/main.yml b/evobackup-client/tasks/main.yml index 4b01a276..c2bfe877 100644 --- a/evobackup-client/tasks/main.yml +++ b/evobackup-client/tasks/main.yml @@ -1,26 +1,31 @@ --- -- ansible.builtin.include: "ssh_key.yml" - tags: - - evobackup_client - - evobackup_client_backup_ssh_key +- name: Install evobackup client components + ansible.builtin.include: "install.yml" -- ansible.builtin.include: "jail.yml" - tags: - - evobackup_client - - evobackup_client_jail +### This is commented because supposedly non-functionnal -- ansible.builtin.include: "upload_scripts.yml" - tags: - - evobackup_client - - evobackup_client_backup_scripts +# - ansible.builtin.include: "ssh_key.yml" +# tags: +# - evobackup_client +# - evobackup_client_backup_ssh_key -- ansible.builtin.include: "open_ssh_ports.yml" - tags: - - evobackup_client - - evobackup_client_backup_firewall +# - ansible.builtin.include: "jail.yml" +# tags: +# - evobackup_client +# - evobackup_client_jail -- ansible.builtin.include: "verify_ssh.yml" - tags: - - evobackup_client - - evobackup_client_backup_hosts +# - ansible.builtin.include: "upload_scripts.yml" +# tags: +# - evobackup_client +# - evobackup_client_backup_scripts + +# - ansible.builtin.include: "open_ssh_ports.yml" +# tags: +# - evobackup_client +# - evobackup_client_backup_firewall + +# - ansible.builtin.include: "verify_ssh.yml" +# tags: +# - evobackup_client +# - evobackup_client_backup_hosts diff --git a/evobackup-client/templates/update-evobackup-canary.sh.j2 b/evobackup-client/templates/update-evobackup-canary.sh.j2 new file mode 100644 index 00000000..e9df9dbd --- /dev/null +++ b/evobackup-client/templates/update-evobackup-canary.sh.j2 @@ -0,0 +1,3 @@ +#!/bin/sh + +update-evobackup-canary --who {{ evobackup_client__update_canary_who | mandatory }} diff --git a/evobackup-client/templates/zzz_evobackup.default.sh.j2 b/evobackup-client/templates/zzz_evobackup.default.sh.j2 deleted file mode 100644 index fd2be5b4..00000000 --- a/evobackup-client/templates/zzz_evobackup.default.sh.j2 +++ /dev/null @@ -1,305 +0,0 @@ -#!/bin/sh -# Careful, the zzz_evobackup template was last updated on 2020/06/08 -# -# Script Evobackup client -# See https://gitea.evolix.org/evolix/evobackup -# -# Author: Gregory Colpart -# Contributors: -# Romain Dessort -# Benoît Série -# Tristan Pilat -# Victor Laborie -# Jérémy Lecour -# -# Licence: AGPLv3 -# -# /!\ DON'T FORGET TO SET "MAIL" and "SERVERS" VARIABLES - -# Fail on unassigned variables -set -u - -##### Configuration ################################################### - -# email adress for notifications -MAIL={{ evobackup_client__mail }} - -# list of hosts (hostname or IP) and SSH port for Rsync -SERVERS="{% for host in evobackup_client__hosts %}{{ host.name }}:{{ host.port }}{% if loop.index != loop.length %} {% endif %}{% endfor %}" - -# Should we fallback on servers when the first is unreachable ? -SERVERS_FALLBACK={{ evobackup_client__servers_fallback }} - -# timeout (in seconds) for SSH connections -SSH_CONNECT_TIMEOUT=${SSH_CONNECT_TIMEOUT:-30} - -## We use /home/backup : feel free to use your own dir -LOCAL_BACKUP_DIR="{{ evobackup_client__backup_path }}" - -# You can set "linux" or "bsd" manually or let it choose automatically -SYSTEM=$(uname | tr '[:upper:]' '[:lower:]') - -# Change these 2 variables if you have more than one backup cron -PIDFILE="{{ evobackup_client__pid_path }}" -LOGFILE="{{ evobackup_client__log_path }}" - -## Enable/Disable tasks -LOCAL_TASKS=${LOCAL_TASKS:-1} -SYNC_TASKS=${SYNC_TASKS:-1} - -##### SETUP AND FUNCTIONS ############################################# - -BEGINNING=$(/bin/date +"%d-%m-%Y ; %H:%M") - -# shellcheck disable=SC2174 -mkdir -p -m 700 ${LOCAL_BACKUP_DIR} - -PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin - -## lang = C for english outputs -export LANGUAGE=C -export LANG=C - -## Force umask -umask 077 - -## Initialize variable to store SSH connection errors -SERVERS_SSH_ERRORS="" - -# Call test_server with "HOST:PORT" string -# It will return with 0 if the server is reachable. -# It will return with 1 and a message on stderr if not. -test_server() { - item=$1 - # split HOST and PORT from the input string - host=$(echo "${item}" | cut -d':' -f1) - port=$(echo "${item}" | cut -d':' -f2) - - # Test if the server is accepting connections - ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" -i {{ evobackup_client__root_key_path }} "${host}" -p "${port}" -t "exit" - # shellcheck disable=SC2181 - if [ $? = 0 ]; then - # SSH connection is OK - return 0 - else - # SSH connection failed - new_error=$(printf "Failed to connect to \`%s' within %s seconds" "${item}" "${SSH_CONNECT_TIMEOUT}") - SERVERS_SSH_ERRORS=$(printf "%s\\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d') - - return 1 - fi -} -# Call pick_server with an optional positive integer to get the nth server in the list. -pick_server() { - increment=${1:-0} - list_length=$(echo "${SERVERS}" | wc -w) - - if [ "${increment}" -ge "${list_length}" ]; then - # We've reached the end of the list - new_error="No more server available" - SERVERS_SSH_ERRORS=$(printf "%s\\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d') - - # Log errors to stderr - printf "%s\\n" "${SERVERS_SSH_ERRORS}" >&2 - # Log errors to logfile - printf "%s\\n" "${SERVERS_SSH_ERRORS}" >> $LOGFILE - return 1 - fi - - # Extract the day of month, without leading 0 (which would give an octal based number) - today=$(date +%e) - # A salt is useful to randomize the starting point in the list - # but stay identical each time it's called for a server (based on hostname). - salt=$(hostname | cksum | cut -d' ' -f1) - # Pick an integer between 0 and the length of the SERVERS list - # It changes each day - item=$(( (today + salt + increment) % list_length )) - # cut starts counting fields at 1, not 0. - field=$(( item + 1 )) - - echo "${SERVERS}" | cut -d' ' -f${field} -} - -## Verify other evobackup process and kill if needed -if [ -e "${PIDFILE}" ]; then - pid=$(cat "${PIDFILE}") - # Does process still exist ? - if kill -0 "${pid}" 2>/dev/null; then - # Killing the childs of evobackup. - for ppid in $(pgrep -P "${pid}"); do - kill -9 "${ppid}"; - done - # Then kill the main PID. - kill -9 "${pid}" - printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\\n" >&2 - else - rm -f ${PIDFILE} - fi -fi -echo "$$" > ${PIDFILE} -# shellcheck disable=SC2064 -trap "rm -f ${PIDFILE}" EXIT - - -##### LOCAL BACKUP #################################################### - -if [ "${LOCAL_TASKS}" = "1" ]; then - ## Dump system and kernel versions - uname -a > ${LOCAL_BACKUP_DIR}/uname - - ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) - for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do - mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr} - traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1 - done - - ## Dump process with ps - ps auwwx >${LOCAL_BACKUP_DIR}/ps.out - - if [ "${SYSTEM}" = "linux" ]; then - ## Dump network connections with ss - ss -taupen > ${LOCAL_BACKUP_DIR}/netstat.out - - ## List Debian packages - dpkg -l > ${LOCAL_BACKUP_DIR}/packages - dpkg --get-selections > ${LOCAL_BACKUP_DIR}/packages.getselections - apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available - - ## Dump MBR / table partitions - disks=$(lsblk -l | grep disk | grep -v -E '(drbd|fd[0-9]+)' | awk '{print $1}') - for disk in ${disks}; do - dd if="/dev/${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)" - fdisk -l "/dev/${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${disk}" 2>&1 - done - cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions - - ## Dump iptables - if [ -x /sbin/iptables ]; then - { /sbin/iptables -L -n -v; /sbin/iptables -t filter -L -n -v; } > ${LOCAL_BACKUP_DIR}/iptables.txt - fi - - ## Dump findmnt(8) output - FINDMNT_BIN=$(command -v findmnt) - if [ -x "${FINDMNT_BIN}" ]; then - ${FINDMNT_BIN} > ${LOCAL_BACKUP_DIR}/findmnt.txt - fi - else - ## Dump network connections with netstat - netstat -finet -atn > ${LOCAL_BACKUP_DIR}/netstat.out - - ## List OpenBSD packages - pkg_info -m > ${LOCAL_BACKUP_DIR}/packages - - ## Dump MBR / table partitions - disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions - - ## Dump pf infos - pfctl -sa > ${LOCAL_BACKUP_DIR}/pfctl-sa.txt - - fi - - ## Dump rights - #getfacl -R /var > ${LOCAL_BACKUP_DIR}/rights-var.txt - #getfacl -R /etc > ${LOCAL_BACKUP_DIR}/rights-etc.txt - #getfacl -R /usr > ${LOCAL_BACKUP_DIR}/rights-usr.txt - #getfacl -R /home > ${LOCAL_BACKUP_DIR}/rights-home.txt - -fi - -##### REMOTE BACKUP ################################################### - -n=0 -server="" -if [ "${SERVERS_FALLBACK}" = "1" ]; then - # We try to find a suitable server - while :; do - server=$(pick_server "${n}") - test $? = 0 || exit 2 - - if test_server "${server}"; then - break - else - server="" - n=$(( n + 1 )) - fi - done -else - # we force the server - server=$(pick_server "${n}") -fi - -SSH_SERVER=$(echo "${server}" | cut -d':' -f1) -SSH_PORT=$(echo "${server}" | cut -d':' -f2) - -HOSTNAME=$(hostname) - -if [ "${SYSTEM}" = "linux" ]; then - rep="/bin /boot /lib /opt /sbin /usr /srv" -else - rep="/bsd /bin /sbin /usr" -fi - - -if [ "${SYNC_TASKS}" = "1" ]; then - # /!\ DO NOT USE COMMENTS in the rsync command /!\ - # It breaks the command and destroys data, simply remove (or add) lines. - - # Remote shell command - RSH_COMMAND="ssh -i {{ evobackup_client__root_key_path }} -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'" - - # ignore check because we want it to split the different arguments to $rep - # shellcheck disable=SC2086 - rsync -avzh --stats --delete --delete-excluded --force --ignore-errors --partial \ - --exclude "lost+found" \ - --exclude ".nfs.*" \ - --exclude "/var/log" \ - --exclude "/var/log/evobackup*" \ - --exclude "/var/lib/mysql" \ - --exclude "/var/lib/postgres" \ - --exclude "/var/lib/postgresql" \ - --exclude "/var/lib/sympa" \ - --exclude "/var/lib/metche" \ - --exclude "/var/run" \ - --exclude "/var/lock" \ - --exclude "/var/state" \ - --exclude "/var/apt" \ - --exclude "/var/cache" \ - --exclude "/usr/src" \ - --exclude "/usr/doc" \ - --exclude "/usr/share/doc" \ - --exclude "/usr/obj" \ - --exclude "dev" \ - --exclude "/var/spool/postfix" \ - --exclude "/var/lib/amavis/amavisd.sock" \ - --exclude "/var/lib/munin/*tmp*" \ - --exclude "/var/lib/php5" \ - --exclude "/var/spool/squid" \ - --exclude "/var/lib/elasticsearch" \ - --exclude "/var/lib/amavis/tmp" \ - --exclude "/var/lib/clamav/*.tmp" \ - --exclude "/home/mysqltmp" \ - --exclude "/var/lib/php/sessions" \ - ${rep} \ - /etc \ - /root \ - /var \ - -e "${RSH_COMMAND}" \ - "root@${SSH_SERVER}:/var/backup/" \ - | tail -30 >> $LOGFILE -fi - -##### REPORTING ####################################################### - -END=$(/bin/date +"%d-%m-%Y ; %H:%M") - -printf "EvoBackup - %s - START %s ON %s (LOCAL_TASKS=%s SYNC_TASKS=%s)\\n" \ - "${HOSTNAME}" "${BEGINNING}" "${SSH_SERVER}" "${LOCAL_TASKS}" "${SYNC_TASKS}" \ - >> $LOGFILE - -printf "EvoBackup - %s - STOP %s ON %s (LOCAL_TASKS=%s SYNC_TASKS=%s)\\n" \ - "${HOSTNAME}" "${END}" "${SSH_SERVER}" "${LOCAL_TASKS}" "${SYNC_TASKS}" \ - >> $LOGFILE - -tail -10 $LOGFILE | \ - mail -s "[info] EvoBackup - Client ${HOSTNAME}" \ - ${MAIL} From 1d5415237ce12906ab9da0c04dcb8078d584fc20 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 30 Apr 2024 17:16:22 +0200 Subject: [PATCH 37/40] sort CHANGELOG --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b98d4d1..0fcf0833 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,16 +30,16 @@ The **patch** part is incremented if multiple releases happen the same month ### Fixed -* certbot: Fix HAProxy renewal hook * certbot: Fix HAPEE renewal hook +* certbot: Fix HAProxy renewal hook +* evolinux-base/logcheck: fix conf patch, journal check was not disabled when asked +* fail2ban: SQLite purge script didn't vacuum as expected + error when vacuum cannot be done * keepalived: Fix tasks that use file instead of copy * memcached: Fix conditions not properly writen (installation was always in multi-instance mode) -* fail2ban: SQLite purge script didn't vacuum as expected + error when vacuum cannot be done * nagios-nrpe: create /etc/bash_completion.d if missing +* openvpn: install packages manually, because openbsd_pkg module is broken since OpenBSD 7.4 with the version of Ansible we currently use * packweb: fix old bug (2017!) .orig file created by module patch and taken in account by ProFTPd * redis: replace inline argument with environment variable for the password -* evolinux-base/logcheck: fix conf patch, journal check was not disabled when asked -* openvpn: install packages manually, because openbsd_pkg module is broken since OpenBSD 7.4 with the version of Ansible we currently use ### Removed From e3746d18fbf2493e0ea7e42cdf43aa911040a378 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 30 Apr 2024 17:38:14 +0200 Subject: [PATCH 38/40] proftpd: remove whitelist block if feature is disabled --- CHANGELOG.md | 2 ++ proftpd/tasks/accounts.yml | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fcf0833..46fc8e67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ The **patch** part is incremented if multiple releases happen the same month ### Added +proftpd: optional configuration of IP whitelists per groups of users + ### Changed * autosysadmin-agent: upstream release 24.03.2 diff --git a/proftpd/tasks/accounts.yml b/proftpd/tasks/accounts.yml index 11b2f60d..08f069b5 100644 --- a/proftpd/tasks/accounts.yml +++ b/proftpd/tasks/accounts.yml @@ -61,7 +61,7 @@ tags: - proftpd -- name: Whitelist ip for users (SFTP) +- name: IP Whitelists for SFTP users are present ansible.builtin.blockinfile: dest: /etc/proftpd/conf.d/sftp.conf marker: "# {mark} ANSIBLE MANAGED BLOCK - Whitelist ip for users" @@ -82,6 +82,14 @@ notify: restart proftpd when: proftpd_sftp_enable_user_whitelist | bool +- name: IP Whitelists for SFTP users are absent + ansible.builtin.blockinfile: + dest: /etc/proftpd/conf.d/sftp.conf + marker: "# {mark} ANSIBLE MANAGED BLOCK - Whitelist ip for users" + state: absent + notify: restart proftpd + when: not (proftpd_sftp_enable_user_whitelist | bool) + - name: Allow keys for SFTP account ansible.builtin.template: dest: "/etc/proftpd/sftp.authorized_keys/{{ _proftpd_account.name }}" From 2ad2ae852140bd4a5070b07e0884d4b4dde42ff0 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 30 Apr 2024 17:38:26 +0200 Subject: [PATCH 39/40] amend CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46fc8e67..8533c916 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,7 @@ proftpd: optional configuration of IP whitelists per groups of users ### Removed -* docker-host: Removed setting docker_conf_use_iptables (iptable usage forced to true) +* docker-host: Removed `docker_conf_use_iptables` variable (iptable usage forced to true) ### Security From 6ee1e609acc11a9b7db08d5cf606defb199bd3fe Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 30 Apr 2024 17:41:15 +0200 Subject: [PATCH 40/40] Release 24.04 --- CHANGELOG.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8533c916..06a03cfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,18 @@ The **patch** part is incremented if multiple releases happen the same month ### Added +### Changed + +### Fixed + +### Removed + +### Security + +## [24.04] 2024-04-30 + +### Added + proftpd: optional configuration of IP whitelists per groups of users ### Changed @@ -47,8 +59,6 @@ proftpd: optional configuration of IP whitelists per groups of users * docker-host: Removed `docker_conf_use_iptables` variable (iptable usage forced to true) -### Security - ## [24.03] 2024-03-01 ### Added