From 5ae3f040980c6fc730db6a6d79ad90fb24c6b94f Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sat, 18 Apr 2020 10:40:28 +0200 Subject: [PATCH 1/9] exclude build directory in vagrant rsync --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index c544987..e7764fe 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -6,7 +6,7 @@ vagrantfile = File.join("#{Dir.home}", '.VagrantFile') load File.expand_path(vagrantfile) if File.exists?(vagrantfile) Vagrant.configure('2') do |config| - config.vm.synced_folder "./", "/vagrant", type: "rsync", rsync__exclude: [ '.vagrant', '.git' ] + config.vm.synced_folder "./", "/vagrant", type: "rsync", rsync__exclude: [ '.vagrant', '.git', 'build' ] config.ssh.shell="/bin/sh" config.vm.provider :libvirt do |libvirt| From 7a13a42ad2d45a5ab6498f889e4b8548dab75366 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sat, 18 Apr 2020 19:09:52 +0200 Subject: [PATCH 2/9] Create empty directory in the default tmp directory --- lib/bkctld-rm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bkctld-rm b/lib/bkctld-rm index 5e4f8fe..73b643c 100755 --- a/lib/bkctld-rm +++ b/lib/bkctld-rm @@ -48,7 +48,7 @@ delete_inc_ext4() { fi mkdir --parents "${LOCKDIR}" && echo $$ > ${lock_file} || error "Failed to acquire lock file '${lock_file}'" - empty=$(mktemp -d --suffix ".${$}" bkctld.XXXXX) + empty=$(mktemp -d --suffix ".${$}" --tmpdir bkctld.XXXXX) # shellcheck disable=SC2064 trap "rm -f ${lock_file}; rmdir ${empty}" 0 From 723c8511e12c4b1817618fe661e8495c60a82aa7 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sat, 18 Apr 2020 19:10:19 +0200 Subject: [PATCH 3/9] Don't forget to log the start time! --- lib/bkctld-rm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/bkctld-rm b/lib/bkctld-rm index 73b643c..320e258 100755 --- a/lib/bkctld-rm +++ b/lib/bkctld-rm @@ -30,6 +30,8 @@ delete_inc_ext4() { inc_path=$(inc_path "${jail_name}" "${inc_name}") + start=$(current_time) + lock_file="${LOCKDIR}/rm-global.lock" if [ -f "${lock_file}" ]; then # Get Process ID from the lock file From 096fe95b2fcefde58a52876ba127ce74791feb74 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sat, 18 Apr 2020 19:11:08 +0200 Subject: [PATCH 4/9] bkctld-rm: fix lock file management logic was badly implemented --- lib/bkctld-rm | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/bkctld-rm b/lib/bkctld-rm index 320e258..c8cbfb5 100755 --- a/lib/bkctld-rm +++ b/lib/bkctld-rm @@ -36,17 +36,21 @@ delete_inc_ext4() { if [ -f "${lock_file}" ]; then # Get Process ID from the lock file pid=$(cat "${lock_file}") - if kill -0 ${pid} 2> /dev/null; then - # Kill the children - pkill -9 --parent "${pid}" - # Kill the parent - kill -9 "${pid}" - # Remove the lock file - rm -f ${lock_file} - warning "Process ${pid} has been killed. Only one ${0} can run in parallel, the latest wins." + if [ -n "${pid}" ]; then + if kill -0 ${pid} 2> /dev/null; then + # Kill the children + pkill -9 --parent "${pid}" + # Kill the parent + kill -9 "${pid}" + warning "Process ${pid} has been killed. Only one ${0} can run in parallel, the latest wins." + else + warning "Process not found at PID \`${pid}'. Ignoring lock file \`${lock_file}'." + fi else - error "Empty lockfile '${lock_file}'. It should contain a PID." + error "Empty lockfile \`${lock_file}'. It should contain a PID." fi + # Remove the lock file + rm -f ${lock_file} fi mkdir --parents "${LOCKDIR}" && echo $$ > ${lock_file} || error "Failed to acquire lock file '${lock_file}'" @@ -60,6 +64,8 @@ delete_inc_ext4() { rsync --archive --delete "${empty}/" "${inc_path}/" rmdir "${inc_path}/" fi + # Remove the lock file + rm -f ${lock_file} end=$(current_time) notice "${jail_name}: inc '${inc_name}' has been deleted [${start}/${end}]" From f66d832d3a6882ceb4e95e4c4a569883b697a24f Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 19 Apr 2020 00:16:30 +0200 Subject: [PATCH 5/9] properly call subcommands --- lib/bkctld-check-incs | 2 +- lib/bkctld-check-last-incs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bkctld-check-incs b/lib/bkctld-check-incs index 22cf1a8..a378bb5 100755 --- a/lib/bkctld-check-incs +++ b/lib/bkctld-check-incs @@ -8,7 +8,7 @@ LIBDIR="$(dirname $0)" && . "${LIBDIR}/includes" # default return value is 0 (succes) rc=0 # loop for each configured jail -for jail_name in $(bkctld list); do +for jail_name in $("${LIBDIR}/bkctld-list"); do incs_policy_file=$(current_jail_incs_policy_file "${jail_name}") # Today in seconds from epoch diff --git a/lib/bkctld-check-last-incs b/lib/bkctld-check-last-incs index 87218a2..9f64d5b 100755 --- a/lib/bkctld-check-last-incs +++ b/lib/bkctld-check-last-incs @@ -8,7 +8,7 @@ LIBDIR="$(dirname $0)" && . "${LIBDIR}/includes" # default return value is 0 (succes) rc=0 # loop for each found jail -for jail_name in $(bkctld list); do +for jail_name in $("${LIBDIR}/bkctld-list"); do incs_policy_file=$(current_jail_incs_policy_file "${jail_name}") if [ -n "${incs_policy_file}" ]; then From 32f242fe9c035c2521bc9992c8fbcd963c3a6a9f Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 19 Apr 2020 09:55:43 +0200 Subject: [PATCH 6/9] Reorganize temp files and lock files Temp files/dirs all share a common prefix, so we can delete them all effectively Lock file is managed globally for bkctld-rm. No need for a lock file for bkctld-inc anymore since there is a check on the inc directory before starting. --- lib/bkctld-inc | 71 ++++++++++-------------------- lib/bkctld-rm | 117 ++++++++++++++++++++++++++++--------------------- lib/includes | 19 ++++++++ 3 files changed, 107 insertions(+), 100 deletions(-) diff --git a/lib/bkctld-inc b/lib/bkctld-inc index 3a0f0b8..3b25095 100755 --- a/lib/bkctld-inc +++ b/lib/bkctld-inc @@ -8,65 +8,33 @@ LIBDIR="$(dirname $0)" && . "${LIBDIR}/includes" create_inc_btrfs() { - jail_name=$1 - inc_name=$2 + jail_name=${1:?} + inc_name=${2:?} jail_path=$(jail_path "${jail_name}") inc_path=$(inc_path "${jail_name}" "${inc_name}") - # The lock file prevents from starting a new copy when one is already being done - lock_file="${LOCKDIR}/inc-${jail_name}-${inc_name}.lock" - if [ -f "${lock_file}" ]; then - warning "${jail_name}: skipping '${inc_name}', it is already being created." + if dry_run; then + echo "[dry-run] btrfs subvolume snapshot of ${jail_path} to ${inc_path}" else - ( - start=$(current_time) - mkdir --parents "${LOCKDIR}" && touch "${lock_file}" - # shellcheck disable=SC2064 - trap "rm -f ${lock_file}" 0 - - if dry_run; then - echo "[dry-run] btrfs subvolume snapshot of ${jail_path} to ${inc_path}" - else - mkdir --parents "$(dirname "${inc_path}")" - # create a btrfs readonly snapshot from the jail - /bin/btrfs subvolume snapshot -r "${jail_path}" "${inc_path}" | debug - fi - - end=$(current_time) - notice "${jail_name}: inc '${inc_name}' has been created [${start}/${end}]" - ) + mkdir --parents "$(dirname "${inc_path}")" + # create a btrfs readonly snapshot from the jail + /bin/btrfs subvolume snapshot -r "${jail_path}" "${inc_path}" | debug fi } create_inc_ext4() { - jail_name=$1 - inc_name=$2 + jail_name=${1:?} + inc_name=${2:?} jail_path=$(jail_path "${jail_name}") inc_path=$(inc_path "${jail_name}" "${inc_name}") - # The lock file prevents from starting a new copy when one is already being done - lock_file="${LOCKDIR}/inc-${jail_name}-${inc_name}.lock" - if [ -f "${lock_file}" ]; then - warning "${jail_name}: skipping '${inc_name}', it is already being created." + if dry_run; then + echo "[dry-run] copy of ${jail_path} to ${inc_path}" else - ( - start=$(current_time) - mkdir --parents "${LOCKDIR}" && touch "${lock_file}" - # shellcheck disable=SC2064 - trap "rm -f ${lock_file}" 0 - - if dry_run; then - echo "[dry-run] copy of ${jail_path} to ${inc_path}" - else - mkdir --parents "$(dirname "${inc_path}")" - # create a copy of the jail with hard links - cp --archive --link --one-file-system "${jail_path}/" "${inc_path}" - fi - - end=$(current_time) - notice "${jail_name}: in '${inc_name}' has been created [${start}/${end}]" - ) + mkdir --parents "$(dirname "${inc_path}")" + # create a copy of the jail with hard links + cp --archive --link --one-file-system "${jail_path}/" "${inc_path}" fi } @@ -79,17 +47,22 @@ for jail_name in $(jails_list); do # If no incs policy is found, we don't create incs if [ -n "${incs_policy_file}" ]; then - # If not incs directory is found, we don't create incs + # If no incs directory is found, we don't create incs if [ ! -d "${inc_path}" ]; then + start=$(current_time) + if is_btrfs "${jail_path}"; then create_inc_btrfs "${jail_name}" "${inc_name}" else create_inc_ext4 "${jail_name}" "${inc_name}" fi + + end=$(current_time) + notice "${jail_name}: \`${inc_name}' has been created [${start}/${end}]" else - warning "${jail_name}: skipping ${inc_name}, it already exists." + warning "${jail_name}: skipping \`${inc_name}', it already exists." fi else - warning "${jail_name}: skipping ${inc_name}, incs policy not found." + notice "${jail_name}: skipping \`${inc_name}', incs policy not found." fi done diff --git a/lib/bkctld-rm b/lib/bkctld-rm index c8cbfb5..c9782b3 100755 --- a/lib/bkctld-rm +++ b/lib/bkctld-rm @@ -7,32 +7,9 @@ # shellcheck source=./includes LIBDIR="$(dirname $0)" && . "${LIBDIR}/includes" -delete_inc_btrfs() { - jail_name=$1 - inc_name=$2 +kill_or_clean_lockfile() { + lock_file=${1:-} - inc_path=$(inc_path "${jail_name}" "${inc_name}") - - start=$(current_time) - - if dry_run; then - echo "[dry-run] delete btrfs subvolume ${inc_path}" - else - /bin/btrfs subvolume delete "${inc_path}" | debug - fi - - end=$(current_time) - notice "${jail_name}: inc '${inc_name}' has been deleted [${start}/${end}]" -} -delete_inc_ext4() { - jail_name=$1 - inc_name=$2 - - inc_path=$(inc_path "${jail_name}" "${inc_name}") - - start=$(current_time) - - lock_file="${LOCKDIR}/rm-global.lock" if [ -f "${lock_file}" ]; then # Get Process ID from the lock file pid=$(cat "${lock_file}") @@ -52,59 +29,97 @@ delete_inc_ext4() { # Remove the lock file rm -f ${lock_file} fi +} +incs_to_delete() { + jail_name=${1:?} + incs_policy_file=${2:?} - mkdir --parents "${LOCKDIR}" && echo $$ > ${lock_file} || error "Failed to acquire lock file '${lock_file}'" - empty=$(mktemp -d --suffix ".${$}" --tmpdir bkctld.XXXXX) - # shellcheck disable=SC2064 - trap "rm -f ${lock_file}; rmdir ${empty}" 0 + incs_policy_keep_file=$(new_tmp_file "${jail_name}.incs_policy_keep") + incs_list_file=$(new_tmp_file "${jail_name}.incs_list") + + # loop for each line in jail configuration + for incs_policy_line in $(grep "^\+" ${incs_policy_file}); do + # inc date in ISO format + incs_policy_date=$(relative_date ${incs_policy_line}) + echo ${incs_policy_date} >> "${incs_policy_keep_file}" + done + for inc_name in $(incs_list "${jail_name}"); do + echo "${inc_name}" >> ${incs_list_file} + done + + # shellcheck disable=SC2046 + incs_to_delete=$(grep -v -f "${incs_policy_keep_file}" "${incs_list_file}") + + rm -f "${incs_policy_keep_file}" "${incs_list_file}" + + echo ${incs_to_delete} +} +delete_inc_btrfs() { + jail_name=$1 + inc_name=$2 + + inc_path=$(inc_path "${jail_name}" "${inc_name}") if dry_run; then - echo "[dry-run] delete ${inc_path} with rsync from ${empty}" + echo "[dry-run] delete btrfs subvolume ${inc_path}" else + /bin/btrfs subvolume delete "${inc_path}" | debug + fi +} +delete_inc_ext4() { + jail_name=$1 + inc_name=$2 + + inc_path=$(inc_path "${jail_name}" "${inc_name}") + + if dry_run; then + echo "[dry-run] delete ${inc_path} with rsync from empty directory" + else + empty=$(new_tmp_dir "empty") rsync --archive --delete "${empty}/" "${inc_path}/" rmdir "${inc_path}/" + rmdir "${empty}" fi - # Remove the lock file - rm -f ${lock_file} - end=$(current_time) - notice "${jail_name}: inc '${inc_name}' has been deleted [${start}/${end}]" } +lock_file="${LOCKDIR}/rm-global.lock" +# shellcheck disable=SC2064 +trap "rm -f ${lock_file}; cleanup_tmp;" 0 + +kill_or_clean_lockfile "${lock_file}" +new_lock_file "${lock_file}" + for jail_name in $(jails_list); do incs_policy_file=$(current_jail_incs_policy_file ${jail_name}) - - # If not incs policy if found, we don't remove incs + # If no incs policy is found, we don't remove incs if [ -n "${incs_policy_file}" ]; then - incs_policy_keep_file="$(mktemp)" - incs_list_file="$(mktemp)" - # shellcheck disable=SC2064 - trap "rm -f ${incs_policy_keep_file} ${incs_list_file}" 0 - - # loop for each line in jail configuration - for incs_policy_line in $(grep "^\+" ${incs_policy_file}); do - # inc date in ISO format - incs_policy_date=$(relative_date ${incs_policy_line}) - echo ${incs_policy_date} >> "${incs_policy_keep_file}" - done - for inc_name in $(incs_list "${jail_name}"); do - echo "${inc_name}" >> ${incs_list_file} - done # shellcheck disable=SC2046 - incs_to_delete=$(grep -v -f "${incs_policy_keep_file}" "${incs_list_file}") + incs_to_delete=$(incs_to_delete "${jail_name}" "${incs_policy_file}") if [ -n "${incs_to_delete}" ]; then debug "${jail_name}: incs to be deleted : $(echo "${incs_to_delete}" | tr '\n', ',' | sed 's/,$//')." for inc_name in ${incs_to_delete}; do + start=$(current_time) + inc_path=$(inc_path "${jail_name}" "${inc_name}") if is_btrfs "${inc_path}"; then delete_inc_btrfs "${jail_name}" "${inc_name}" else delete_inc_ext4 "${jail_name}" "${inc_name}" fi + + end=$(current_time) + notice "${jail_name}: inc \`${inc_name}' has been deleted [${start}/${end}]" done else notice "${jail_name}: no inc to be deleted." fi + else + notice "${jail_name}: skipping jail because incs policy is missing." fi done + +# Remove the lock file and cleanup tmp files +rm -f "${lock_file}" +cleanup_tmp diff --git a/lib/includes b/lib/includes index ebf7b37..7398f62 100755 --- a/lib/includes +++ b/lib/includes @@ -177,6 +177,25 @@ relative_date() { echo ${past_date} } +new_tmp_file() { + name=${1:-} + + mktemp --tmpdir=/tmp "bkctld.${$}.${name}.XXXXX" +} +new_tmp_dir() { + name=${1:-} + + mktemp --directory --tmpdir=/tmp "bkctld.${$}.${name}.XXXXX" +} +cleanup_tmp() { + find /tmp -name "bkctld.${$}.*" -delete +} +new_lock_file() { + lock_file=${1:-} + lock_dir=$(dirname "${lock_file}") + + mkdir --parents "${lock_dir}" && echo $$ > ${lock_file} || error "Failed to acquire lock file '${lock_file}'" +} setup_jail_chroot() { jail_name=${1:?} From 478703d96e938410dea59e417a54f78dd92224e7 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 19 Apr 2020 09:56:52 +0200 Subject: [PATCH 7/9] No space before colons in log messages --- lib/includes | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/includes b/lib/includes index 7398f62..8ecf986 100755 --- a/lib/includes +++ b/lib/includes @@ -47,18 +47,18 @@ notice() { warning() { msg="${1:-$(cat /dev/stdin)}" - tty -s && echo "WARNING : ${msg}" >&2 + tty -s && echo "WARNING: ${msg}" >&2 if [ "${LOGLEVEL}" -ge 4 ]; then - tty -s || echo "WARNING : ${msg}" >&2 + tty -s || echo "WARNING: ${msg}" >&2 logger -t bkctld -p daemon.warning "${msg}" fi } error() { msg="${1:-$(cat /dev/stdin)}" - tty -s && echo "ERROR : ${msg}" >&2 + tty -s && echo "ERROR: ${msg}" >&2 if [ "${LOGLEVEL}" -ge 5 ]; then - tty -s || echo "ERROR : ${msg}" >&2 + tty -s || echo "ERROR: ${msg}" >&2 logger -t bkctld -p daemon.error "${msg}" fi exit 1 From 1846be4e45e8a5f7b44c8034edf1a25f55149a1a Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 19 Apr 2020 10:05:20 +0200 Subject: [PATCH 8/9] better quotes --- lib/bkctld-check | 4 ++-- lib/bkctld-ip | 10 +++++----- lib/bkctld-key | 4 ++-- lib/bkctld-port | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/bkctld-check b/lib/bkctld-check index 1bc31c4..4cd5054 100755 --- a/lib/bkctld-check +++ b/lib/bkctld-check @@ -21,7 +21,7 @@ if [ -b "${BACKUP_DISK}" ]; then cryptsetup isLuks "${BACKUP_DISK}" if [ "$?" -eq 0 ]; then if [ ! -b '/dev/mapper/backup' ]; then - echo "Luks disk ${BACKUP_DISK} is not mounted !\n" + echo "Luks disk \`${BACKUP_DISK}' is not mounted !\n" echo "cryptsetup luksOpen ${BACKUP_DISK} backup" exit 2 fi @@ -31,7 +31,7 @@ if [ -b "${BACKUP_DISK}" ]; then # Verify that it's mounted and writable findmnt --source ${BACKUP_DISK} -O rw > /dev/null if [ "$?" -ne 0 ]; then - echo "Backup disk ${BACKUP_DISK} is not mounted (or read-only) !\n" + echo "Backup disk \`${BACKUP_DISK}' is not mounted (or read-only) !\n" echo "mount ${BACKUP_DISK} /backup" exit 2 fi diff --git a/lib/bkctld-ip b/lib/bkctld-ip index a71d144..8bb8054 100755 --- a/lib/bkctld-ip +++ b/lib/bkctld-ip @@ -31,18 +31,18 @@ else new_ips="0.0.0.0/0" else existing_ips=$("${LIBDIR}/bkctld-ip" "${jail_name}") - new_ips=$(echo "${existing_ips}" "${ip}" | xargs -n1 | grep -v "0.0.0.0/0" | sort | uniq) + new_ips=$(echo ${existing_ips} ${ip} | xargs -n1 | grep -v "0.0.0.0/0" | sort | uniq) fi allow_users="AllowUsers" - for ip in ${new_ips}; do - allow_users="${allow_users} root@${ip}" + for new_ip in ${new_ips}; do + allow_users="${allow_users} root@${new_ip}" done if grep -q -E "^AllowUsers" "${jail_sshd_config}"; then sed -i "s~^AllowUsers .*~${allow_users}~" "${jail_sshd_config}" else - error "${jail_name}: No 'AllowUsers' directive found in '${jail_sshd_config}'" + error "${jail_name}: No \`AllowUsers' directive found in \`${jail_sshd_config}'" fi - notice "${jail_name}: IP whitelist updated to ${ip}" + notice "${jail_name}: IP whitelist updated with \`${ip}'" "${LIBDIR}/bkctld-reload" "${jail_name}" "${LIBDIR}/bkctld-firewall" "${jail_name}" fi diff --git a/lib/bkctld-key b/lib/bkctld-key index b7b1ed7..2c01ef8 100755 --- a/lib/bkctld-key +++ b/lib/bkctld-key @@ -22,8 +22,8 @@ if [ -z "${keyfile}" ]; then cat "${jail_path}/${AUTHORIZED_KEYS}" fi else - test -r "${keyfile}" || error "${jail_name}: SSH key '${keyfile}' is missing or is not readable." + test -r "${keyfile}" || error "${jail_name}: SSH key \`${keyfile}' is missing or is not readable." cat "${keyfile}" > "${jail_path}/${AUTHORIZED_KEYS}" chmod 600 "${jail_path}/${AUTHORIZED_KEYS}" - notice "${jail_name}: SSH key has been updated with ${keyfile}" + notice "${jail_name}: SSH key has been updated with \`${keyfile}'" fi diff --git a/lib/bkctld-port b/lib/bkctld-port index 1ee67c7..0b3c514 100755 --- a/lib/bkctld-port +++ b/lib/bkctld-port @@ -29,7 +29,7 @@ else fi sed -i "s/^Port .*/Port ${port}/" "${jail_sshd_config}" - notice "${jail_name}: port has been updated to ${port}" + notice "${jail_name}: port has been updated to \`${port}'" "${LIBDIR}/bkctld-reload" "${jail_name}" "${LIBDIR}/bkctld-firewall" "${jail_name}" From 0bad1a2cba41865954234ace30d51fea53bdd721 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 19 Apr 2020 10:07:35 +0200 Subject: [PATCH 9/9] Release 2.2.2 --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c75b813..c022e3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +## [2.2.2] - 2020-04-19 + +### Changed + +* Reorganize temp files and lock files + +### Fixed + +* Properly call subcommands in bkctld-check-incs and bkctld-check-last-incs +* Log start time in bkctld-rm + ## [2.2.1] - 2020-04-18 ### Changed