From 5020ca4bfddb85bd295b557da52f2b85fc7e3b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 11 Oct 2020 18:46:15 +0200 Subject: [PATCH 1/2] improve documentation --- README.md | 94 ++++++++++++--------- bkctld.8 | 6 +- bkctld.conf.5 | 4 +- docs/configuration.md | 4 +- docs/incrementals.md | 12 +-- docs/usage.md | 24 +++--- evobackup-incl.5 => evobackup-incs-policy.5 | 8 +- lib/includes | 4 +- tpl/{inc.tpl => incs-policy.tpl} | 0 9 files changed, 86 insertions(+), 70 deletions(-) rename evobackup-incl.5 => evobackup-incs-policy.5 (92%) rename tpl/{inc.tpl => incs-policy.tpl} (100%) diff --git a/README.md b/README.md index 8ad80e7..411f7db 100644 --- a/README.md +++ b/README.md @@ -43,18 +43,70 @@ This architecture is as secure as SSH, Rsync, chroot and iptables are. See the [installation guide](docs/install.md) for instructions. +## Server-side usage + +See [docs/usage.md](docs/usage.md). + +The man(1) page, in troff(7) language, can be generated with pandoc: + +~~~ +pandoc -f markdown \ + -t man usage.md \ + --template default.man \ + -V title=bkctld \ + -V section=8 \ + -V date="$(date '+%d %b %Y')" \ + -V footer="$(git describe --tags)" \ + -V header="bkctld man page" +~~~ + +~~~ +git clone https://gitea.evolix.org/evolix/evobackup.git +cd evobackup +man ./docs/bkctld.8 +~~~ + +#### Client-side usage + +As bkctld creates a dedicated SSH server, everything that connects over SSH can push files to the jail (Rsync, SFTP…). + +Example with Rsync : + +~~~ +rsync -av -e "ssh -p SSH_PORT" /home/ root@SERVER_NAME:/var/backup/home/ +~~~ + +An example synchronization script is present in `zzz_evobackup`, +clone the evobackup repository and read the **CLIENT CONFIGURATION** +section of the manual. + + ## Testing -You can deploy test environments with Vagrant : +You can manage test environments with Vagrant. + +There are 4 preconfigured environments : +* Debian stetch with ext4 +* Debian stetch with btrfs +* Debian buster with ext4 +* Debian buster with btrfs + +You can start them all with : ~~~ vagrant up ~~~ +… or just the one you want : + +~~~ +vagrant up buster-btrfs +~~~ + ### Deployment Run `vagrant rsync-auto` in a terminal for automatic synchronization of -your local code with Vagrant VM : +your local code with Vagrant environments : ~~~ vagrant rsync-auto @@ -77,40 +129,4 @@ vagrant@buster-btrfs $ sudo -i root@buster-btrfs # bats /vagrant/test/*.bats ~~~ -You should shellcheck your bats files, but with shellcheck > 0.4.6, because the 0.4.0 version doesn't support bats syntax. - -## Usage - -See [docs/usage.md](docs/usage.md). - -The man(1) page, in troff(7) language, can be generated with pandoc: - -~~~ -pandoc -f markdown \ - -t man usage.md \ - --template default.man \ - -V title=bkctld \ - -V section=8 \ - -V date="$(date '+%d %b %Y')" \ - -V footer="$(git describe --tags)" \ - -V header="bkctld man page" -~~~ - -#### Client configuration - -You can backup various systems in the evobackup jails : Linux, BSD, -Windows, macOS. The only need Rsync or an SFTP client. - -~~~ -rsync -av -e "ssh -p SSH_PORT" /home/ root@SERVER_NAME:/var/backup/home/ -~~~ - -An example synchronization script is present in `zzz_evobackup`, -clone the evobackup repository and read the **CLIENT CONFIGURATION** -section of the manual. - -~~~ -git clone https://gitea.evolix.org/evolix/evobackup.git -cd evobackup -man ./docs/bkctld.8 -~~~ +You should [shellcheck](https://www.shellcheck.net) your bats files, but with shellcheck > 0.4.6, because the 0.4.0 version doesn't support bats syntax. diff --git a/bkctld.8 b/bkctld.8 index a162bb8..13a38a8 100644 --- a/bkctld.8 +++ b/bkctld.8 @@ -31,7 +31,7 @@ using hard links or BTRFS snapshots. (So they can not be affected by the client), which backups are kept over time can be configured in the jail's nominal -.Xr evobackup-incl 5 +.Xr evobackup-incs-policy 5 configuration file. .Pp A large enough volume must be mounted on @@ -131,7 +131,7 @@ He can then create the jail: .Ed .Pp And override the default -.Xr evobackup-incl 5 +.Xr evobackup-incs-policy 5 rules .Bd -literal -offset indent # $EDITOR /etc/evobackup/CLIENT_HOST_NAME @@ -169,7 +169,7 @@ can be found in the .Xr rsync 1 , .Xr ssh-keygen 1 , .Xr bkctld 5 , -.Xr evobackup-incl 5 , +.Xr evobackup-incs-policy 5 , .Xr chroot 8 , .Xr cron 8 , .Xr sshd 8 diff --git a/bkctld.conf.5 b/bkctld.conf.5 index 3e44bd3..f2b9cd0 100644 --- a/bkctld.conf.5 +++ b/bkctld.conf.5 @@ -28,7 +28,7 @@ The following variables may be defined: .Bl -tag -width Ds .It Va CONFDIR Directory where -.Xr evobackup-incl 5 +.Xr evobackup-incs-policy 5 files are kept. It's default value is .Pa /etc/evobackup/ . @@ -63,5 +63,5 @@ will not automatically update the firewall. .El .Sh SEE ALSO .Xr bash 1 , -.Xr evobackup-incl 5 , +.Xr evobackup-incs-policy 5 , .Xr bkctld 8 diff --git a/docs/configuration.md b/docs/configuration.md index 7b9456e..2ed904d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -32,7 +32,7 @@ The following variables may be defined: *CONFDIR* > Directory where -> evobackup-incl(5) +> evobackup-incs-policy(5) > files are kept. > It's default value is > */etc/evobackup/*. @@ -80,7 +80,7 @@ The following variables may be defined: # SEE ALSO bash(1), -evobackup-incl(5), +evobackup-incs-policy(5), bkctld(8) OpenBSD 6.4 - December 28, 2018 diff --git a/docs/incrementals.md b/docs/incrementals.md index 046fc7d..bf0ec19 100644 --- a/docs/incrementals.md +++ b/docs/incrementals.md @@ -1,8 +1,8 @@ -EVOBACKUP-INCL(5) - File Formats Manual +evobackup-incs-policy(5) - File Formats Manual # NAME -**evobackup-incl** - incremental backup configuration +**evobackup-incs-policy** - incremental backup configuration # SYNOPSIS @@ -13,7 +13,7 @@ EVOBACKUP-INCL(5) - File Formats Manual Located by default in */etc/evobackup/*, each -**evobackup-incl** +**evobackup-incs-policy** file is named after the bkctld(8) backup for which the rules it contains must apply. @@ -31,9 +31,9 @@ character are comments and are ignored. The order of the rules does not matter. Evobackups that do not have their nominal -**evobackup-incl** +**evobackup-incs-policy** file use the default rules defined in -*/usr/share/bkctld/inc.tpl* +*/usr/share/bkctld/incs-policy.tpl* # EXAMPLES @@ -75,6 +75,6 @@ Keep backups of the last 4 days and the first day of the last 2 months: bkctld(8), cron(8), -*/etc/evobackup/tpl/inc.tpl* +*/etc/evobackup/tpl/incs-policy.tpl* OpenBSD 6.4 - December 28, 2018 diff --git a/docs/usage.md b/docs/usage.md index afcbfa1..f0778ae 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -27,14 +27,14 @@ in it's chroot(8) (using the root account). -Prior backups are stored incrementally outside of the +Dated copies (_incs_) of the data are stored outside of the chroot(8) using ln(1) hard links or BTRFS snapshots. (So they can not be affected by the client), -which backups are kept over time can be configured in the jail's nominal -evobackup-incl(5) +Old dated copies (_incs_) are deleted every day according to the jail's policy, configured in +evobackup-incs-policy(5) configuration file. A large enough volume must be mounted on @@ -57,7 +57,7 @@ The following operands are available: > Update an evobackup jail -**remove** **all** | *jailname* +**remove** \[-f|--force\] **all** | *jailname* > Remove an evobackup jail @@ -115,11 +115,11 @@ The following operands are available: **inc** -> Generate incremental backups +> Generate a dated copy of the jail **rm** -> Remove old incremental backups +> Remove old dated copies of the jail # FILES @@ -128,9 +128,9 @@ The following operands are available: > Template for > bkctld.conf(5) -*/usr/share/bkctld/incl.tpl* +*/usr/share/bkctld/incs-policy.tpl* -> Default rules for the incremental backups are stored here. +> Default rules for the dated copies retention policy. # EXAMPLES @@ -139,7 +139,7 @@ the backup server administrator will need: * The host name of the client system. -* The public RSA +* The public SSH key ssh(1) key for the "root" @@ -162,7 +162,7 @@ He can then create the jail: # bkctld status And override the default -evobackup-incl(5) +evobackup-incs-policy(5) rules # $EDITOR /etc/evobackup/.d/incs_policy @@ -195,13 +195,13 @@ can be found in the rsync(1), ssh-keygen(1), bkctld(5), -evobackup-incl(5), +evobackup-incs-policy(5), chroot(8), cron(8), sshd(8) # AUTHORS -Victor Laborie +Victor Laborie, Jérémy Lecour and others at Evolix OpenBSD 6.4 - December 27, 2018 diff --git a/evobackup-incl.5 b/evobackup-incs-policy.5 similarity index 92% rename from evobackup-incl.5 rename to evobackup-incs-policy.5 index 48f5453..9161ec3 100644 --- a/evobackup-incl.5 +++ b/evobackup-incs-policy.5 @@ -1,8 +1,8 @@ .Dd December 28, 2018 -.Dt EVOBACKUP-INCL 5 +.Dt evobackup-incs-policy 5 .Os .Sh NAME -.Nm evobackup-incl +.Nm evobackup-incs-policy .Nd incremental backup configuration .Sh SYNOPSIS .D1 +%Y-%m-%d.-%day @@ -31,7 +31,7 @@ The order of the rules does not matter. Evobackups that do not have their nominal .Nm file use the default rules defined in -.Pa /usr/share/bkctld/inc.tpl +.Pa /usr/share/bkctld/incs-policy.tpl .Sh EXAMPLES Keep today's backup: .Bd -literal -offset indent @@ -76,4 +76,4 @@ Keep backups of the last 4 days and the first day of the last 2 months: .Sh SEE ALSO .Xr bkctld 8 , .Xr cron 8 , -.Pa /etc/evobackup/tpl/inc.tpl \ No newline at end of file +.Pa /etc/evobackup/tpl/incs-policy.tpl diff --git a/lib/includes b/lib/includes index 9b0c211..e40e81f 100755 --- a/lib/includes +++ b/lib/includes @@ -366,8 +366,8 @@ setup_jail_config() { info "4 - Copie default sshd_config" install -m 0640 "${sshd_config_tpl}" "${jail_sshd_config}" - inctpl="${TPLDIR}/inc.tpl" - test -f "${LOCALTPLDIR}/inc.tpl" && inctpl="${LOCALTPLDIR}/inc.tpl" + inctpl="${TPLDIR}/incs-policy.tpl" + test -f "${LOCALTPLDIR}/incs-policy.tpl" && inctpl="${LOCALTPLDIR}/incs-policy.tpl" info "5 - Copie default inc configuration" jail_incs_policy_file=$(jail_incs_policy_file "${jail_name}") diff --git a/tpl/inc.tpl b/tpl/incs-policy.tpl similarity index 100% rename from tpl/inc.tpl rename to tpl/incs-policy.tpl -- 2.39.2 From 7d4de721ea13b780e2e723f6d4350d715469f6d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Lecour?= Date: Sun, 11 Oct 2020 23:41:38 +0200 Subject: [PATCH 2/2] new subcommands with backward compatibility --- bkctld | 105 ++++++++++++++++++++++--- bkctld.8 | 6 +- docs/incrementals.md | 2 +- docs/install.md | 6 +- docs/usage.md | 4 +- evobackup-incs-policy.5 | 2 +- lib/{bkctld-inc => bkctld-incs-create} | 0 lib/{bkctld-rm => bkctld-incs-prune} | 0 lib/includes | 6 ++ test/incs.bats | 14 ++-- 10 files changed, 118 insertions(+), 27 deletions(-) rename lib/{bkctld-inc => bkctld-incs-create} (100%) rename lib/{bkctld-rm => bkctld-incs-prune} (100%) diff --git a/bkctld b/bkctld index 07d399a..e00dd12 100755 --- a/bkctld +++ b/bkctld @@ -51,6 +51,7 @@ while :; do exit 0 ;; -f|--force) + # shellcheck disable=SC2034 FORCE=1 ;; *) @@ -64,11 +65,34 @@ done subcommand="${1:-}" case "${subcommand}" in - "inc" | "rm" | "check-jails" | "check-setup" | "stats" | "list") - "${LIBDIR}/bkctld-${subcommand}" + "list") + "${LIBDIR}/bkctld-list" + ;; + "stats") + "${LIBDIR}/bkctld-stats" + ;; + "incs-create") + "${LIBDIR}/bkctld-incs-create" + ;; + "inc") + deprecated_cmd "bkctld inc", "bkctld incs-create" + "${LIBDIR}/bkctld-incs-create" + ;; + "incs-prune") + "${LIBDIR}/bkctld-incs-prune" + ;; + "rm") + deprecated_cmd "bkctld rm", "bkctld incs-prune" + "${LIBDIR}/bkctld-incs-prune" ;; "check") - # backward compatibility + deprecated_cmd "bkctld check", "bkctld jails-check" + "${LIBDIR}/bkctld-check-jails" + ;; + "check-setup") + "${LIBDIR}/bkctld-check-setup" + ;; + "check-jails") "${LIBDIR}/bkctld-check-jails" ;; "check-incs") @@ -91,17 +115,78 @@ case "${subcommand}" in "${LIBDIR}/bkctld-${subcommand}" "${jail_name}" fi ;; - "key" | "port" | "ip") + "ip-add") jail_name="${2:-}" option="${3:-}" - if [ "${jail_name}" = "all" ] || [ -z "${jail_name}" ]; then - show_help - exit 1 - else - "${LIBDIR}/bkctld-${subcommand}" "${jail_name}" "${option}" + "${LIBDIR}/bkctld-ip" "${jail_name}" "${option}" + ;; + "ip-replace") + jail_name="${2:-}" + option="${3:-}" + "${LIBDIR}/bkctld-ip" "${jail_name}" "0.0.0.0/0" + if [ -n "${option}" ]; then + "${LIBDIR}/bkctld-ip" "${jail_name}" "${option}" fi ;; - "start" | "stop" | "reload" | "restart" | "sync" | "update" | "remove" | "firewall" | "upgrade-config") + "ip-list") + jail_name="${2:-}" + "${LIBDIR}/bkctld-ip" "${jail_name}" + ;; + "ip") + jail_name="${2:-}" + option="${3:-}" + if [ -n "${option}" ]; then + if [ "${option}" = "0.0.0.0/0" ] || [ "${option}" = "all" ]; then + deprecated_cmd "bkctld ip ${jail_name} ${option}", "bkctld ip-replace ${jail_name}" + else + deprecated_cmd "bkctld ip ${jail_name} ${option}", "bkctld ip-add ${jail_name} ${option}" + fi + "${LIBDIR}/bkctld-ip" "${jail_name}" "${option}" + else + deprecated_cmd "bkctld ip ${jail_name}", "bkctld ip-list ${jail_name}" + "${LIBDIR}/bkctld-ip" "${jail_name}" + fi + ;; + "key-list") + jail_name="${2:-}" + "${LIBDIR}/bkctld-key" "${jail_name}" + ;; + "key-replace") + jail_name="${2:-}" + option="${3:-}" + "${LIBDIR}/bkctld-key" "${jail_name}" "${option}" + ;; + "key-add") + error "\`bkctld key-add' command is not available yet!" + ;; + "key") + jail_name="${2:-}" + option="${3:-}" + if [ -n "${option}" ]; then + deprecated_cmd "bkctld key ${jail_name} ${option}", "bkctld key-replace ${jail_name} ${option}" + "${LIBDIR}/bkctld-key" "${jail_name}" "${option}" + else + deprecated_cmd "bkctld key ${jail_name}", "bkctld key-list ${jail_name}" + "${LIBDIR}/bkctld-key" "${jail_name}" + fi + ;; + "port") + jail_name="${2:-}" + option="${3:-}" + "${LIBDIR}/bkctld-port" "${jail_name}" "${option}" + ;; + "archive") + error "\`bkctld archive' command is not available yet!" + ;; + "remove") + jail_name="${2:-}" + if [ "${jail_name}" = "all" ]; then + error "\`bkctld remove all' is too dangerous." + else + "${LIBDIR}/bkctld-remove" "${jail_name}" + fi + ;; + "start" | "stop" | "reload" | "restart" | "sync" | "update" | "firewall" | "upgrade-config") jail_name="${2:-}" if [ "${jail_name}" = "all" ]; then for jail in $("${LIBDIR}/bkctld-list"); do diff --git a/bkctld.8 b/bkctld.8 index 13a38a8..9edd304 100644 --- a/bkctld.8 +++ b/bkctld.8 @@ -124,8 +124,8 @@ in He can then create the jail: .Bd -literal -offset indent # bkctld init CLIENT_HOST_NAME -# bkctld key CLIENT_HOST_NAME /root/CLIENT_HOST_NAME.pub -# bkctld ip CLIENT_HOST_NAME CLIENT_IP_ADDRESS +# bkctld key-add CLIENT_HOST_NAME /root/CLIENT_HOST_NAME.pub +# bkctld ip-add CLIENT_HOST_NAME CLIENT_IP_ADDRESS # bkctld start CLIENT_HOST_NAME # bkctld status CLIENT_HOST_NAME .Ed @@ -176,4 +176,4 @@ can be found in the .Sh AUTHORS .An Victor Laborie .\" .Sh CAVEATS -.\" .Sh BUGS \ No newline at end of file +.\" .Sh BUGS diff --git a/docs/incrementals.md b/docs/incrementals.md index bf0ec19..877989c 100644 --- a/docs/incrementals.md +++ b/docs/incrementals.md @@ -20,7 +20,7 @@ backup for which the rules it contains must apply. The rules it defines decide which incremental backups are kept when running - # bkctld rm + # bkctld incs-prune Each line defines a single rule. The first part of the rule describes when the backup was taken, diff --git a/docs/install.md b/docs/install.md index bfb6d7d..f63e0d1 100644 --- a/docs/install.md +++ b/docs/install.md @@ -61,11 +61,11 @@ Edit the root crontab ~~~ # $editor /etc/cron.d/bkctld + MAILTO=alert4@evolix.net -+ 30 11 * * * root /usr/sbin/bkctld inc && /usr/sbin/bkctld rm ++ 30 11 * * * root /usr/sbin/bkctld incs-create && /usr/sbin/bkctld incs-prune + 30 23 * * * root /usr/share/scripts/check-incs.sh 1> /dev/null ~~~ ## Notes If you want mutiples backups in a day (1 per hour maximum) you can -run `bkctld inc` multiples times, if you want to keep incremental -backups **for ever**, just don't run `bkctld rm`. +run `bkctld incs-create` multiples times, if you want to keep incremental +backups **for ever**, just don't run `bkctld incs-prune`. diff --git a/docs/usage.md b/docs/usage.md index f0778ae..1609778 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -156,8 +156,8 @@ the backup server administrator will need: He can then create the jail: # bkctld init - # bkctld key /root/.pub - # bkctld ip + # bkctld key-add /root/.pub + # bkctld ip-add # bkctld start # bkctld status diff --git a/evobackup-incs-policy.5 b/evobackup-incs-policy.5 index 9161ec3..0f841ee 100644 --- a/evobackup-incs-policy.5 +++ b/evobackup-incs-policy.5 @@ -17,7 +17,7 @@ backup for which the rules it contains must apply. .Pp The rules it defines decide which incremental backups are kept when running .Bd -literal -offset indent -# bkctld rm +# bkctld incs-prune .Ed .Pp Each line defines a single rule. diff --git a/lib/bkctld-inc b/lib/bkctld-incs-create similarity index 100% rename from lib/bkctld-inc rename to lib/bkctld-incs-create diff --git a/lib/bkctld-rm b/lib/bkctld-incs-prune similarity index 100% rename from lib/bkctld-rm rename to lib/bkctld-incs-prune diff --git a/lib/includes b/lib/includes index e40e81f..32626d5 100755 --- a/lib/includes +++ b/lib/includes @@ -111,6 +111,12 @@ error() { fi exit ${rc} } +deprecated_cmd() { + old=${1:?} + new=${2:?} + + warning "\`${old}' is deprecated, use \`${new}' instead" +} dry_run() { test "$DRY_RUN" = "1" diff --git a/test/incs.bats b/test/incs.bats index 380d990..3542718 100644 --- a/test/incs.bats +++ b/test/incs.bats @@ -11,7 +11,7 @@ load test_helper } @test "Normal inc creation" { - /usr/lib/bkctld/bkctld-inc + /usr/lib/bkctld/bkctld-incs-create if is_btrfs "/backup"; then # On a btrfs filesystem, the inc should be a btrfs volume @@ -27,7 +27,7 @@ load test_helper @test "Normal inc creation (with old incs policy)" { mv "${CONFDIR}/${JAILNAME}.d/incs_policy" "${CONFDIR}/${JAILNAME}" - /usr/lib/bkctld/bkctld-inc + /usr/lib/bkctld/bkctld-incs-create if is_btrfs "/backup"; then # On a btrfs filesystem, the inc should be a btrfs volume @@ -46,7 +46,7 @@ load test_helper # … and old file rm -f "${CONFDIR}/${JAILNAME}" - /usr/lib/bkctld/bkctld-inc + /usr/lib/bkctld/bkctld-incs-create run test -d "${INCSPATH}/${INC_NAME}" assert_failure @@ -60,8 +60,8 @@ load test_helper recent_inc_path="${INCSPATH}/${INC_NAME}" # Create the inc, then run 'rm' - /usr/lib/bkctld/bkctld-inc - /usr/lib/bkctld/bkctld-rm + /usr/lib/bkctld/bkctld-incs-create + /usr/lib/bkctld/bkctld-incs-prune # Recent inc should be present run test -d "${recent_inc_path}" @@ -78,9 +78,9 @@ load test_helper older_inc_path="${INCSPATH}/${older_inc_name}" # Create the inc, rename it to make it older, then run 'rm' - /usr/lib/bkctld/bkctld-inc + /usr/lib/bkctld/bkctld-incs-create mv "${recent_inc_path}" "${older_inc_path}" - /usr/lib/bkctld/bkctld-rm + /usr/lib/bkctld/bkctld-incs-prune # Older inc should be removed run test -d "${older_inc_path}" -- 2.39.2