From bdb84e809f9cdba04a7530be1d111d1df3fcf7f0 Mon Sep 17 00:00:00 2001 From: David Prevot Date: Wed, 29 Jun 2022 13:41:31 +0200 Subject: [PATCH 01/23] =?UTF-8?q?CI:=20Don=E2=80=99t=20rely=20on=20/tmp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /tmp is mounted noexec, so can’t be used to run scripts. --- .Jenkinsfile | 6 +++--- .drone.yml | 13 +------------ 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/.Jenkinsfile b/.Jenkinsfile index 0b3ae83..6cabf61 100644 --- a/.Jenkinsfile +++ b/.Jenkinsfile @@ -5,7 +5,7 @@ pipeline { agent { docker { image 'evolix/gbp:bullseye' - args '-u root --privileged -v /tmp:/tmp' + args '-u root --privileged' } } when { @@ -29,8 +29,8 @@ pipeline { steps { script { sh 'echo Dummy line to remove once something actually happens.' - /* No crendentials yet - sh 'rsync -avP /tmp/bkctld/ droneci@pub.evolix.net:/home/droneci/bkctld/' + /* No crendentials yet. + sh 'rsync -avP bkctld* droneci@pub.evolix.net:/home/droneci/bkctld/' */ } } diff --git a/.drone.yml b/.drone.yml index cd183e5..97a9e97 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,9 +15,6 @@ steps: - mk-build-deps --install --remove debian/control - git clean --force - gbp buildpackage -us -uc - volumes: - - name: tmp - path: /tmp when: branch: - debian @@ -31,16 +28,8 @@ steps: key: from_secret: drone_private_key target: /home/droneci/bkctld/ - source: /tmp/bkctld/ + source: ../bkctld* delete: true - volumes: - - name: tmp - path: /tmp when: branch: - debian - -volumes: -- name: tmp - host: - path: /tmp From 7901fd1950dc3a36ca8778cb567393d8fa818380 Mon Sep 17 00:00:00 2001 From: David Prevot Date: Wed, 29 Jun 2022 14:03:03 +0200 Subject: [PATCH 02/23] Jenkins CI: Improve clean up --- .Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.Jenkinsfile b/.Jenkinsfile index 6cabf61..a8de7f2 100644 --- a/.Jenkinsfile +++ b/.Jenkinsfile @@ -14,7 +14,7 @@ pipeline { steps { script { sh 'mk-build-deps --install --remove debian/control' - sh 'rm -rf source' + sh 'rm -rf {source,*.gz,*.bz2,*.xz,*.deb,*.dsc,*.changes,*.buildinfo,lintian.txt}' sh "gbp clone --debian-branch=$GIT_BRANCH $GIT_URL source" sh 'cd source && git checkout $GIT_BRANCH && gbp buildpackage -us -uc' } From ba4629bee70553bd572bfbc3d0be9610677b1597 Mon Sep 17 00:00:00 2001 From: David Prevot Date: Thu, 30 Jun 2022 15:53:39 +0200 Subject: [PATCH 03/23] server/README.md: tfix --- server/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/README.md b/server/README.md index 8ad80e7..9d93fa9 100644 --- a/server/README.md +++ b/server/README.md @@ -4,7 +4,7 @@ Bkctld (aka server-side evobackup) bkctld helps you manage the receiving side of a backup infrastructure. It is licensed under the AGPLv3. -With bkctld you create and manage "jails". They contain a chrooted and dedicated SSH server, with it's own TCP port and optionnaly it's own set of iptables rules. +With bkctld you create and manage "jails". They contain a chrooted and dedicated SSH server, with its own TCP port and optionally its own set of iptables rules. With bkctld you can have hundreds of jails, one for each client to push its data (using Rsync/SFTP). Each client can only see its own data. @@ -30,9 +30,7 @@ This volume can also be encrypted with **LUKS**. ## Security considerations -The client obviously has access to its uploaded data (in the chroot), but the timestamped copies are outside the chroot, to reduce the risk or complete backup erasure from a compromised client. - -Since the client connects to the backup server with root, it can mess with the jail and destroy the data. But the timestamped copies are out of reach because outside of the chroot. +The client obviously has access to its uploaded data (in the chroot), but the timestamped copies are outside the chroot, to reduce the risk of complete backup erasure from a compromised client. It means that **if the client server is compromised**, an attacker can destroy the latest copy of the backed up data, but not the timestamped copies. And **if the backup server is compromised** an attacker has complete access to all the backup data (inside and outside the jails), but they don't have any access to the client. @@ -77,6 +75,8 @@ vagrant@buster-btrfs $ sudo -i root@buster-btrfs # bats /vagrant/test/*.bats ~~~ +[comment]: <> (* pour vim) + You should shellcheck your bats files, but with shellcheck > 0.4.6, because the 0.4.0 version doesn't support bats syntax. ## Usage @@ -99,7 +99,7 @@ pandoc -f markdown \ #### Client configuration You can backup various systems in the evobackup jails : Linux, BSD, -Windows, macOS. The only need Rsync or an SFTP client. +Windows, macOS. The only need is Rsync or an SFTP client. ~~~ rsync -av -e "ssh -p SSH_PORT" /home/ root@SERVER_NAME:/var/backup/home/ From f0581fee47951a68f899939de38a0fefe5fac1e4 Mon Sep 17 00:00:00 2001 From: David Prevot Date: Mon, 11 Jul 2022 14:39:50 +0200 Subject: [PATCH 04/23] CI: Drop .git directory that was not present during first build --- .Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.Jenkinsfile b/.Jenkinsfile index a8de7f2..583a3ad 100644 --- a/.Jenkinsfile +++ b/.Jenkinsfile @@ -14,7 +14,7 @@ pipeline { steps { script { sh 'mk-build-deps --install --remove debian/control' - sh 'rm -rf {source,*.gz,*.bz2,*.xz,*.deb,*.dsc,*.changes,*.buildinfo,lintian.txt}' + sh 'rm -rf {source,*.gz,*.bz2,*.xz,*.deb,*.dsc,*.changes,*.buildinfo,lintian.txt,.git}' sh "gbp clone --debian-branch=$GIT_BRANCH $GIT_URL source" sh 'cd source && git checkout $GIT_BRANCH && gbp buildpackage -us -uc' } From aef2637c1e72ec3fabfb0c1b90145d294d93e9dd Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 20 Jul 2022 13:43:33 +0200 Subject: [PATCH 05/23] check-setup: get minifirewall version from internal variable there is no other backward compatible way : * really old version don't have a know version * some versions used to display the version on each command, but it is removed. * the VERSION variable seems to be the most forward-compatible way --- server/CHANGELOG.md | 2 ++ server/lib/bkctld-check-setup | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index cf3a7e4..ccc06fc 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +* check-setup: get minifirewall version from internal variable (there is no other backward compatible way) + ### Deprecated ### Removed diff --git a/server/lib/bkctld-check-setup b/server/lib/bkctld-check-setup index 8bbcda9..5f9983a 100755 --- a/server/lib/bkctld-check-setup +++ b/server/lib/bkctld-check-setup @@ -29,7 +29,7 @@ fi # Check if the firewall file is sourced minifirewall_config=/etc/default/minifirewall -minifirewall_version=$(/etc/init.d/minifirewall status | head -1 | cut -d ' ' -f 3) +minifirewall_version=$(grep -E -o "^VERSION=(\S+)" /etc/init.d/minifirewall | head -1 | cut -d '=' -f 2 | tr -d "'" | tr -d '"') if [ -n "${FIREWALL_RULES}" ] \ && [ -r "${FIREWALL_RULES}" ] \ From 1891c98f57556ad99c35c5556141cade26416ac4 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 20 Jul 2022 14:23:13 +0200 Subject: [PATCH 06/23] check-setup: use findmnt with mountpoint instead of target --- server/lib/bkctld-check-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/bkctld-check-setup b/server/lib/bkctld-check-setup index 5f9983a..6e8ac21 100755 --- a/server/lib/bkctld-check-setup +++ b/server/lib/bkctld-check-setup @@ -16,7 +16,7 @@ output="" # Verify backup partition is mounted and writable -findmnt -O rw --target "${BACKUP_PARTITION}" > /dev/null +findmnt -O rw --mountpoint "${BACKUP_PARTITION}" > /dev/null if [ "$?" -ne 0 ]; then nb_crit=$((nb_crit + 1)) output="${output}CRITICAL - Backup disk \`/backup' is not mounted (or read-only) !\n" From a3ca2f0f68e66c221d1a615522157532fb475ebf Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 20 Jul 2022 14:24:12 +0200 Subject: [PATCH 07/23] check-setup: check minifirewall version only if minifirewall is present --- server/lib/bkctld-check-setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/lib/bkctld-check-setup b/server/lib/bkctld-check-setup index 6e8ac21..137c49c 100755 --- a/server/lib/bkctld-check-setup +++ b/server/lib/bkctld-check-setup @@ -29,11 +29,12 @@ fi # Check if the firewall file is sourced minifirewall_config=/etc/default/minifirewall -minifirewall_version=$(grep -E -o "^VERSION=(\S+)" /etc/init.d/minifirewall | head -1 | cut -d '=' -f 2 | tr -d "'" | tr -d '"') if [ -n "${FIREWALL_RULES}" ] \ && [ -r "${FIREWALL_RULES}" ] \ && [ -f "${minifirewall_config}" ]; then + minifirewall_version=$(grep -E -o "^VERSION=(\S+)" /etc/init.d/minifirewall | head -1 | cut -d '=' -f 2 | tr -d "'" | tr -d '"') + if [ -n "${minifirewall_version}" ] && dpkg --compare-versions "${minifirewall_version}" ge "22.03"; then # Minifirewall 22.03+ includes files automatically nb_ok=$((nb_ok + 1)) From e7b7f50d9dee811b1756d9db72d00ee131b02d1f Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 20 Jul 2022 14:31:22 +0200 Subject: [PATCH 08/23] server: release 22.07 --- server/CHANGELOG.md | 10 ++++++++-- server/lib/includes | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index ccc06fc..0c21896 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -10,8 +10,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -* check-setup: get minifirewall version from internal variable (there is no other backward compatible way) - ### Deprecated ### Removed @@ -20,6 +18,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +## [22.07] - 2022-07-20 + +### Changed + +* check-setup: check minifirewall version only if minifirewall is present +* check-setup: get minifirewall version from internal variable (there is no other backward compatible way) +* check-setup: use findmnt with mountpoint instead of target + ## [22.06] - 2022-06-28 ### Added diff --git a/server/lib/includes b/server/lib/includes index 91321a3..3653c40 100755 --- a/server/lib/includes +++ b/server/lib/includes @@ -6,7 +6,7 @@ [ -f /etc/default/bkctld ] && . /etc/default/bkctld -VERSION="22.04" +VERSION="22.07" LIBDIR=${LIBDIR:-/usr/lib/bkctld} CONFDIR="${CONFDIR:-/etc/evobackup}" From c80881debfb29a08f490795293e988487bababba Mon Sep 17 00:00:00 2001 From: Alexis Ben Miloud--Josselin Date: Wed, 24 Aug 2022 14:40:30 +0200 Subject: [PATCH 09/23] Exclure datadir MongoDB --- client/zzz_evobackup | 1 + 1 file changed, 1 insertion(+) diff --git a/client/zzz_evobackup b/client/zzz_evobackup index 9beccfb..5ab2d45 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -420,6 +420,7 @@ sync_tasks() { --exclude "/var/lib/metche" \ --exclude "/var/lib/munin/*tmp*" \ --exclude "/var/db/munin/*.tmp" \ + --exclude "/var/lib/mongodb" \ --exclude "/var/lib/mysql" \ --exclude "/var/lib/php5" \ --exclude "/var/lib/php/sessions" \ From 89580f29291aa4e04286ac74942f436500e038af Mon Sep 17 00:00:00 2001 From: William Hirigoyen Date: Fri, 26 Aug 2022 10:58:48 +0200 Subject: [PATCH 10/23] Fix variable reading in case VAR is specified multiple times in file. --- server/lib/includes | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/lib/includes b/server/lib/includes index 3653c40..3ed7bcd 100755 --- a/server/lib/includes +++ b/server/lib/includes @@ -445,7 +445,7 @@ read_variable() { pattern="^\s*${var_name}=.+" - grep --extended-regexp --only-matching "${pattern}" "${file}" | tail -1 | cut -d= -f2 + grep --extended-regexp --only-matching "${pattern}" "${file}" | tail -1 | cut -d= -f2 | head -1 } read_numerical_variable() { @@ -454,5 +454,5 @@ read_numerical_variable() { pattern="^\s*${var_name}=-?[0-9]+" - grep --extended-regexp --only-matching "${pattern}" "${file}" | tail -1 | cut -d= -f2 + grep --extended-regexp --only-matching "${pattern}" "${file}" | tail -1 | cut -d= -f2 | head -1 } From 5739b8afe2375c2dcb9230fe08849344fa64838e Mon Sep 17 00:00:00 2001 From: William Hirigoyen Date: Fri, 26 Aug 2022 18:05:14 +0200 Subject: [PATCH 11/23] Revert "Fix variable reading in case VAR is specified multiple times in file." This reverts commit 89580f29291aa4e04286ac74942f436500e038af. --- server/lib/includes | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/lib/includes b/server/lib/includes index 3ed7bcd..3653c40 100755 --- a/server/lib/includes +++ b/server/lib/includes @@ -445,7 +445,7 @@ read_variable() { pattern="^\s*${var_name}=.+" - grep --extended-regexp --only-matching "${pattern}" "${file}" | tail -1 | cut -d= -f2 | head -1 + grep --extended-regexp --only-matching "${pattern}" "${file}" | tail -1 | cut -d= -f2 } read_numerical_variable() { @@ -454,5 +454,5 @@ read_numerical_variable() { pattern="^\s*${var_name}=-?[0-9]+" - grep --extended-regexp --only-matching "${pattern}" "${file}" | tail -1 | cut -d= -f2 | head -1 + grep --extended-regexp --only-matching "${pattern}" "${file}" | tail -1 | cut -d= -f2 } From c769a6e823449ff6e7bb1bd802d51183a7d4793b Mon Sep 17 00:00:00 2001 From: Eric Morino Date: Mon, 5 Sep 2022 14:42:06 +0200 Subject: [PATCH 12/23] Update link to the installation documentation --- server/README.md | 2 +- server/docs/install.md | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/server/README.md b/server/README.md index 9d93fa9..de53c96 100644 --- a/server/README.md +++ b/server/README.md @@ -39,7 +39,7 @@ This architecture is as secure as SSH, Rsync, chroot and iptables are. ## Install -See the [installation guide](docs/install.md) for instructions. +See the [installation guide](https://intra.evolix.net/OutilsInternes/bkctld) for instructions. ## Testing diff --git a/server/docs/install.md b/server/docs/install.md index 5760174..a956ac4 100644 --- a/server/docs/install.md +++ b/server/docs/install.md @@ -2,15 +2,7 @@ ## Install from package -A Debian package is available in the Evolix repository - -~~~ -echo "deb http://pub.evolix.net/ stretch" >> /etc/apt/sources.list -apt update -apt install bkctld -~~~ - -Then edit `/etc/default/bkctld` +The install documentation is [here](https://intra.evolix.net/OutilsInternes/bkctld) ## Instal from sources From f683691853208860bc257616d7a3b31c420425b2 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Thu, 13 Oct 2022 09:09:01 +0200 Subject: [PATCH 13/23] client: tolerate absence of mtr or traceroute --- client/CHANGELOG.md | 1 + client/zzz_evobackup | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/client/CHANGELOG.md b/client/CHANGELOG.md index 4065645..8de2b31 100644 --- a/client/CHANGELOG.md +++ b/client/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * update-evobackup-canary: do not use GNU date, for it to be compatible with OpenBSD * Add AGPL License and README * Script now depends on Bash +* tolerate absence of mtr or traceroute ### Deprecated diff --git a/client/zzz_evobackup b/client/zzz_evobackup index 5ab2d45..45f0588 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -182,9 +182,11 @@ local_tasks() { # rm -rf ${LOCAL_BACKUP_DIR}/pg.*.gz # rm -rf ${LOCAL_BACKUP_DIR}/pg-backup.tar # rm -rf ${LOCAL_BACKUP_DIR}/postgresql/* + ## example with pg_dumpall (warning: you need space in ~postgres) # su - postgres -c "pg_dumpall > ~/pg.dump.bak" # mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/ + ## another method with gzip directly piped # cd /var/lib/postgresql # sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz @@ -265,10 +267,19 @@ local_tasks() { #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls) - for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do - mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr} - traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1 - done + network_targets="8.8.8.8 www.evolix.fr travaux.evolix.net" + mtr_bin=$(command -v mtr) + if [ -n "${mtr_bin}" ]; then + for addr in ${network_targets}; do + ${mtr_bin} -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr} + done + fi + traceroute_bin=$(command -v traceroute) + if [ -n "${traceroute_bin}" ]; then + for addr in ${network_targets}; do + ${traceroute_bin} -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1 + done + fi server_state_dir="${LOCAL_BACKUP_DIR}/server-state" @@ -598,8 +609,6 @@ export LC_ALL=C # Error on unassigned variable set -u -# Fail if a pipeline member returns an error (cf. https://sipb.mit.edu/doc/safe-shell/) -set -o pipefail # Default return-code (0 == succes) rc=0 From aff5dbba9500ec526b0ad4b2fe03f6711d10c100 Mon Sep 17 00:00:00 2001 From: William Hirigoyen Date: Thu, 27 Oct 2022 17:10:14 +0200 Subject: [PATCH 14/23] Add --no-header option for status command. --- server/README.md | 6 ++++++ server/bkctld | 7 ++++++- server/lib/bkctld-status | 2 +- server/lib/includes | 1 + 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/server/README.md b/server/README.md index de53c96..6c9247f 100644 --- a/server/README.md +++ b/server/README.md @@ -49,6 +49,12 @@ You can deploy test environments with Vagrant : vagrant up ~~~ +To destroy Vagrant VMs : + +~~~ +vagrant destroy +~~~ + ### Deployment Run `vagrant rsync-auto` in a terminal for automatic synchronization of diff --git a/server/bkctld b/server/bkctld index 03535d3..2fd07ea 100755 --- a/server/bkctld +++ b/server/bkctld @@ -53,6 +53,9 @@ while :; do -f|--force) export FORCE=1 ;; + --no-header) + export HEADER=0 + ;; *) # Default case: If no more options then break out of the loop. break @@ -116,7 +119,9 @@ case "${subcommand}" in ;; "status") jail_name="${2:-}" - printf '%-30s %-10s %-10s %-25s %-20s\n' 'JAIL NAME' 'STATUS' 'PORT' 'RETENTION (DAY/MONTH)' 'IP' + if [ "${HEADER}" = "1" ]; then + printf '%-30s %-10s %-10s %-25s %-20s\n' 'JAIL NAME' 'STATUS' 'PORT' 'RETENTION (DAY/MONTH)' 'IP' + fi if [ "${jail_name}" = "all" ] || [ -z "${jail_name}" ]; then for jail in $("${LIBDIR}/bkctld-list"); do "${LIBDIR}/bkctld-${subcommand}" "${jail}" diff --git a/server/lib/bkctld-status b/server/lib/bkctld-status index b7cadbb..5546d63 100755 --- a/server/lib/bkctld-status +++ b/server/lib/bkctld-status @@ -1,7 +1,7 @@ #!/bin/sh # # Description: Display status of SSH server -# Usage: status [|all] +# Usage: [--no-header] status [|all] # # shellcheck source=./includes diff --git a/server/lib/includes b/server/lib/includes index 3653c40..6d6d285 100755 --- a/server/lib/includes +++ b/server/lib/includes @@ -29,6 +29,7 @@ CRITICAL="${CRITICAL:-48}" WARNING="${WARNING:-24}" DUC=$(command -v duc-nox || command -v duc) FORCE="${FORCE:-0}" +HEADER="${HEADER:-1}" show_version() { cat < Date: Wed, 26 Oct 2022 09:39:19 +0200 Subject: [PATCH 15/23] More explicit warning for rsync comments/spaces --- client/zzz_evobackup | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/client/zzz_evobackup b/client/zzz_evobackup index 45f0588..28a693f 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -403,12 +403,18 @@ sync_tasks() { update-evobackup-canary --who "${PROGNAME}" - # /!\ DO NOT USE COMMENTS in the rsync command /!\ - # It breaks the command and destroys data, simply remove (or add) lines. - # Remote shell command RSH_COMMAND="ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'" + ################################################################### + # /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ # + ################################################################### + # DO NOT USE COMMENTS in rsync lines # + # DO ADD WHITESPACES AFTER \ in rsync lines # + # It breaks the command and destroys data # + # => Only remove (or add) lines. # + ################################################################### + # ignore check because we want it to split the different arguments to $rep # shellcheck disable=SC2086 rsync --archive \ From 18e056337781f8513d5fdf91ff05cfb610a202e6 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 28 Oct 2022 14:02:26 +0200 Subject: [PATCH 16/23] quotes --- client/zzz_evobackup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/zzz_evobackup b/client/zzz_evobackup index 28a693f..abcde6d 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -271,13 +271,13 @@ local_tasks() { mtr_bin=$(command -v mtr) if [ -n "${mtr_bin}" ]; then for addr in ${network_targets}; do - ${mtr_bin} -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr} + ${mtr_bin} -r "${addr}" > "${LOCAL_BACKUP_DIR}/mtr-${addr}" done fi traceroute_bin=$(command -v traceroute) if [ -n "${traceroute_bin}" ]; then for addr in ${network_targets}; do - ${traceroute_bin} -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1 + ${traceroute_bin} -n "${addr}" > "${LOCAL_BACKUP_DIR}/traceroute-${addr}" 2>&1 done fi From 88a7907fd37c3ca1327c4059f77833fdda552f0f Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 28 Oct 2022 15:48:59 +0200 Subject: [PATCH 17/23] typo --- client/zzz_evobackup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/zzz_evobackup b/client/zzz_evobackup index abcde6d..1b05c18 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -410,7 +410,7 @@ sync_tasks() { # /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ WARNING /!\ # ################################################################### # DO NOT USE COMMENTS in rsync lines # - # DO ADD WHITESPACES AFTER \ in rsync lines # + # DO NOT ADD WHITESPACES AFTER \ in rsync lines # # It breaks the command and destroys data # # => Only remove (or add) lines. # ################################################################### From 2b83cd71bc7627a05a6be6b1e83fbc5f703a8d67 Mon Sep 17 00:00:00 2001 From: Alexis Ben Miloud--Josselin Date: Mon, 14 Nov 2022 11:29:43 +0100 Subject: [PATCH 18/23] postgresql: change wd before dump --- client/zzz_evobackup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/zzz_evobackup b/client/zzz_evobackup index 1b05c18..191e15f 100755 --- a/client/zzz_evobackup +++ b/client/zzz_evobackup @@ -201,9 +201,10 @@ local_tasks() { ## example with compressed PostgreSQL dump for each databases # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/postgresql # chown postgres:postgres ${LOCAL_BACKUP_DIR}/postgresql + # cd /var/lib/postgresql # dbs=$(sudo -u postgres psql -U postgres -lt | awk -F\| '{print $1}' |grep -v template*) - # # for databases in $dbs ; do sudo -u postgres /usr/bin/pg_dump --create -s -U postgres -d $databases | gzip --best -c > ${LOCAL_BACKUP_DIR}/postgresql/$databases.sql.gz ; done + # cd - > /dev/null ## MongoDB From 8ee2aa3b512a9c8b673e0bb533c1b8654f636fa7 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 23 Nov 2022 11:46:33 +0100 Subject: [PATCH 19/23] bkctld-check-canary: new subcommand to check canary files and content --- server/CHANGELOG.md | 2 ++ server/bkctld | 2 +- server/lib/bkctld-check-canary | 59 ++++++++++++++++++++++++++++++++++ server/lib/includes | 8 +++++ 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100755 server/lib/bkctld-check-canary diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 0c21896..35de9dc 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* check-canary: new subcommand to check canary files and content + ### Changed ### Deprecated diff --git a/server/bkctld b/server/bkctld index 2fd07ea..0ee7c87 100755 --- a/server/bkctld +++ b/server/bkctld @@ -67,7 +67,7 @@ done subcommand="${1:-}" case "${subcommand}" in - "inc" | "rm" | "check-jails" | "check-setup" | "stats" | "list") + "inc" | "rm" | "check-jails" | "check-setup" | "check-canary" | "stats" | "list") "${LIBDIR}/bkctld-${subcommand}" ;; "check") diff --git a/server/lib/bkctld-check-canary b/server/lib/bkctld-check-canary new file mode 100755 index 0000000..c2de38a --- /dev/null +++ b/server/lib/bkctld-check-canary @@ -0,0 +1,59 @@ +#!/bin/sh +# +# Description: check canary file +# Usage: check-canary [|all] +# + +# shellcheck source=./includes +LIBDIR="$(dirname $0)" && . "${LIBDIR}/includes" + +return=0 +nb_crit=0 +nb_warn=0 +nb_ok=0 +nb_unkn=0 +output="" + +date=$(date +"%Y-%m-%d") + +# Check each jail status + +check_jail() { + jail_name=$1 + + jail_path=$(jail_path "${jail_name}") + canary_absolute_file="${jail_path}/var/backup/${CANARY_RELATIVE_FILE}" + + if [ -f "${canary_absolute_file}" ]; then + if grep --quiet --fixed-string "${date}" "${canary_absolute_file}"; then + nb_ok=$((nb_ok + 1)) + output="${output}OK - ${jail_name} - entries found for ${date} in ${CANARY_RELATIVE_FILE} file\n" + else + nb_crit=$((nb_crit + 1)) + output="${output}CRITICAL - ${jail_name} - No entry for ${date} in ${CANARY_RELATIVE_FILE} file\n" + [ "${return}" -le 2 ] && return=2 + fi + else + nb_crit=$((nb_crit + 1)) + output="${output}CRITICAL - ${jail_name} - missing ${CANARY_RELATIVE_FILE} file\n" + [ "${return}" -le 2 ] && return=2 + fi +} + +for jail_name in $(jails_list); do + check_jail "${jail_name}" +done + +[ "${return}" -ge 0 ] && header="OK" +[ "${return}" -ge 1 ] && header="WARNING" +[ "${return}" -ge 2 ] && header="CRITICAL" +[ "${return}" -ge 3 ] && header="UNKNOWN" + +printf "%s - %s UNK / %s CRIT / %s WARN / %s OK\n\n" "${header}" "${nb_unkn}" "${nb_crit}" "${nb_warn}" "${nb_ok}" + +printf "${output}" | grep -E "^UNKNOWN" +printf "${output}" | grep -E "^CRITICAL" +printf "${output}" | grep -E "^WARNING" +printf "${output}" | grep -E "^OK" + +exit "${return}" \ No newline at end of file diff --git a/server/lib/includes b/server/lib/includes index 6d6d285..04c5479 100755 --- a/server/lib/includes +++ b/server/lib/includes @@ -20,6 +20,7 @@ LOCKDIR="${LOCKDIR:-/run/lock/bkctld}" ARCHIVESDIR="${ARCHIVESDIR:-${BACKUP_PARTITION}/archives}" INDEX_DIR="${INDEX_DIR:-${BACKUP_PARTITION}/index}" IDX_FILE="${IDX_FILE:-${INDEX_DIR}/bkctld-jails.idx}" +CANARY_RELATIVE_FILE="${CANARY_RELATIVE_FILE:-/zzz_evobackup_canary}" SSHD_PID="${SSHD_PID:-/run/sshd.pid}" SSHD_CONFIG="${SSHD_CONFIG:-/etc/ssh/sshd_config}" AUTHORIZED_KEYS="${AUTHORIZED_KEYS:-/root/.ssh/authorized_keys}" @@ -63,6 +64,13 @@ EOF printf "\n" } +is_quiet() { + test ${QUIET} -eq 1 +} +is_verbose() { + test ${VERBOSE} -eq 1 +} + log_date() { echo "[$(date +"%Y-%m-%d %H:%M:%S")]" } From 860b982556ec851db7f2d82110bdc7d8221facaa Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 23 Nov 2022 12:56:22 +0100 Subject: [PATCH 20/23] better error message --- server/lib/bkctld-stats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/bkctld-stats b/server/lib/bkctld-stats index 8e69f12..e023a14 100755 --- a/server/lib/bkctld-stats +++ b/server/lib/bkctld-stats @@ -15,7 +15,7 @@ ionice -c3 "${DUC}" index -d "${IDX_FILE}" "${JAILDIR}" touch "${INDEX_DIR}/.lastrun.duc" EOF -[ ! -f "${INDEX_DIR}/.lastrun.duc" ] && notice "First run of DUC always in progress ..." && exit 0 +[ ! -f "${INDEX_DIR}/.lastrun.duc" ] && notice "First run of DUC still in progress ..." && exit 0 [ ! -f ${IDX_FILE} ] && error "Index file doesn't exits !" printf "Last update of index file : " From d3c75ab94ea33793c64964edb66bb7f04af1df1c Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Wed, 23 Nov 2022 12:57:28 +0100 Subject: [PATCH 21/23] bkctld-stats: filter active jails and columnize the output --- server/CHANGELOG.md | 2 ++ server/lib/bkctld-stats | 17 +++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 35de9dc..2c7ebd2 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +* stats: filter active jails and columnize the output + ### Deprecated ### Removed diff --git a/server/lib/bkctld-stats b/server/lib/bkctld-stats index e023a14..1f0d2ce 100755 --- a/server/lib/bkctld-stats +++ b/server/lib/bkctld-stats @@ -20,22 +20,24 @@ EOF printf "Last update of index file : " stat --format=%Y "${INDEX_DIR}/.lastrun.duc" | xargs -i -n1 date -R -d "@{}" -echo " " | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $1, $2, $3, $4); }' duc_output=$(mktemp) stat_output=$(mktemp) incs_output=$(mktemp) +jail_patterns_list=$(mktemp) # shellcheck disable=SC2064 -trap "rm ${duc_output} ${incs_output} ${stat_output}" 0 +trap "rm ${duc_output} ${incs_output} ${stat_output} ${jail_patterns_list}" 0 -"${DUC}" ls -d "${IDX_FILE}" "${JAILDIR}" > "${duc_output}" +"${DUC}" ls --database "${IDX_FILE}" "${JAILDIR}" > "${duc_output}" -awk '{ print $2 }' "${duc_output}" | while read jail_name; do +jails_list | sed -e "s/^\(.*\)$/\\\\b\1\\\\b/" > "${jail_patterns_list}" + +grep -f "${jail_patterns_list}" "${duc_output}" | awk '{ print $2 }' | while read jail_name; do jail_path=$(jail_path "${jail_name}") stat --format=%Y "${jail_path}/var/log/lastlog" | xargs -i -n1 date -d "@{}" "+%d-%m-%Y" >> "${stat_output}" - incs_policy_file=$(current_jail_incs_policy_file ${jail_name}) + incs_policy_file=$(current_jail_incs_policy_file "${jail_name}") incs_policy="0" if [ -r "${incs_policy_file}" ]; then days=$(grep "^\+" "${incs_policy_file}" | grep --count "day") @@ -45,4 +47,7 @@ awk '{ print $2 }' "${duc_output}" | while read jail_name; do echo "${incs_policy}" >> "${incs_output}" done -paste "${duc_output}" "${incs_output}" "${stat_output}" | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $2, $1, $3, $4); }' +( + echo " " + paste "${duc_output}" "${incs_output}" "${stat_output}" | awk '{ printf("%s %s %s %s\n", $2, $1, $3, $4); }' +) | column -t From 777b469485f8091f114fcfbe1d9255026051ad0e Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 28 Nov 2022 14:44:19 +0100 Subject: [PATCH 22/23] bkctld check-canary: add tests --- server/test/checks.bats | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/server/test/checks.bats b/server/test/checks.bats index 48c74ea..4dba35e 100644 --- a/server/test/checks.bats +++ b/server/test/checks.bats @@ -252,3 +252,25 @@ OUT assert_failure } # TODO: write many more tests for bkctld-check-incs + +@test "Check-canary fails if a canary file doesn't exist" { + run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}" + assert_equal "$status" "2" + assert_line "CRITICAL - ${JAILNAME} - missing /zzz_evobackup_canary file" +} + +@test "Check-canary fails if a canary is missing today's entries" { + today="$(date +%Y-%m-%d)" + touch "${JAILPATH}/var/backup/zzz_evobackup_canary" + + run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}" + assert_equal "$status" "2" + assert_line "CRITICAL - ${JAILNAME} - No entry for ${today} in /zzz_evobackup_canary file" +} + +@test "Check-canary succeeds if a canary has today's entries" { + echo "$(date "+%FT%T%z") bats-test" >> "${JAILPATH}/var/backup/zzz_evobackup_canary" + + run /usr/lib/bkctld/bkctld-check-canary "${JAILNAME}" + assert_success +} \ No newline at end of file From c8cfbe18aa16e17d0c444401636f2aa502e17f12 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Mon, 28 Nov 2022 14:48:03 +0100 Subject: [PATCH 23/23] server: release 22.11 --- server/CHANGELOG.md | 14 ++++++++++---- server/lib/includes | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 2c7ebd2..3012189 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -8,12 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -* check-canary: new subcommand to check canary files and content - ### Changed -* stats: filter active jails and columnize the output - ### Deprecated ### Removed @@ -22,6 +18,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +## [22.11] - 2022-11-28 + +### Added + +* check-canary: new subcommand to check canary files and content + +### Changed + +* stats: filter active jails and columnize the output + ## [22.07] - 2022-07-20 ### Changed diff --git a/server/lib/includes b/server/lib/includes index 04c5479..59c31c8 100755 --- a/server/lib/includes +++ b/server/lib/includes @@ -6,7 +6,7 @@ [ -f /etc/default/bkctld ] && . /etc/default/bkctld -VERSION="22.07" +VERSION="22.11" LIBDIR=${LIBDIR:-/usr/lib/bkctld} CONFDIR="${CONFDIR:-/etc/evobackup}"