#!/bin/sh # # bkctld is a shell script to create and manage a backup server which will # handle the backup of many servers (clients). # # Author: Victor Laborie # Contributor: Benoît Série , Gregory Colpart , Romain Dessort , Tristan Pilat # Licence: AGPLv3 # set -u usage(){ cat < [options] Subcommands: init Init jail update |all Update jail or all remove |all Remove jail or all start |all Start jail or all stop |all Stop jail or all reload |all Reload jail or all restart |all Restart jail or all sync |all Sync jail or all to another node status [] Print status of (default all jail) key [] Set or get ssh pubic key of port [|auto] Set or get ssh port of ip [|all] Set or get allowed(s) ip(s) of inc Make incremental inc of all jails rm Remove old incremtal inc of all jails check Run check on jails (NRPE output) stats Make and display stats on jails (size, lastconn) EOF } ## logging functions debug() { msg="${1:-$(cat /dev/stdin)}" if [ "${LOGLEVEL}" -ge 7 ]; then echo "${msg}" logger -t bkctld -p daemon.debug "${msg}" fi } info() { msg="${1:-$(cat /dev/stdin)}" if [ "${LOGLEVEL}" -ge 6 ]; then tty -s && echo "${msg}" logger -t bkctld -p daemon.info "${msg}" fi } notice() { msg="${1:-$(cat /dev/stdin)}" tty -s && echo "${msg}" [ "${LOGLEVEL}" -ge 5 ] && logger -t bkctld -p daemon.notice "${msg}" } warning() { msg="${1:-$(cat /dev/stdin)}" tty -s && echo "WARNING : ${msg}" >&2 if [ "${LOGLEVEL}" -ge 4 ]; then tty -s || echo "WARNING : ${msg}" >&2 logger -t bkctld -p daemon.warning "${msg}" fi } error() { msg="${1:-$(cat /dev/stdin)}" tty -s && echo "ERROR : ${msg}" >&2 if [ "${LOGLEVEL}" -ge 5 ]; then tty -s || echo "ERROR : ${msg}" >&2 logger -t bkctld -p daemon.error "${msg}" fi exit 1 } ## check functions check_jail() { jail="${1}" [ -d "${JAILDIR}/${jail}" ] && return 0 return 1 } check_jail_on() { jail="${1}" return=1 if [ -f "${JAILDIR}/${jail}/${SSHD_PID}" ]; then pid=$(cat "${JAILDIR}/${jail}/${SSHD_PID}") ps -p "${pid}" > /dev/null && return=0 fi if [ "${return}" -eq 1 ]; then rm -f "${JAILDIR}/${jail}/${SSHD_PID}" grep -q "${JAILDIR}/${jail}/proc" /proc/mounts && umount --lazy "${JAILDIR}/${jail}/proc/" grep -q "${JAILDIR}/${jail}/dev" /proc/mounts && umount --lazy --recursive "${JAILDIR}/${jail}/dev" fi return "${return}" } ## get functions : get info on jail get_port() { jail="${1}" port=$(grep -E "Port [0-9]+" "${JAILDIR}/${jail}/${SSHD_CONFIG}"|grep -oE "[0-9]+") echo "${port}" } get_key() { jail="${1}" if [ -f "${JAILDIR}/${jail}/${AUTHORIZED_KEYS}" ]; then cat "${JAILDIR}/${jail}/${AUTHORIZED_KEYS}" fi } get_ip() { jail="${1}" grep -E "^AllowUsers" "${JAILDIR}/$jail/${SSHD_CONFIG}"|grep -Eo "root@[^ ]+"| while read allow; do echo "${allow}"|cut -d'@' -f2 done } get_inc() { jail="${1}" inc="0" if [ -f "${CONFDIR}/${jail}" ]; then day=$(grep -c "day" "${CONFDIR}/${jail}") month=$(grep -c "month" "${CONFDIR}/${jail}") inc="${day}/${month}" fi echo "${inc}" } ## set functions : set info on jail set_port() { jail="${1}" port="${2}" if [ "${port}" = "auto" ]; then port=$(grep -h Port "${JAILDIR}"/*/"${SSHD_CONFIG}" 2>/dev/null | grep -Eo "[0-9]+" | sort -n | tail -1) port=$((port+1)) [ "${port}" -le 1 ] && port=2222 fi sed -i "s/^Port .*/Port ${port}/" "${JAILDIR}/$jail/${SSHD_CONFIG}" set_firewall "${jail}" } set_key() { jail="${1}" keyfile="${2}" [ -e "${keyfile}" ] || error "Keyfile ${keyfile} dosen't exist !" cat "${keyfile}" > "${JAILDIR}/${jail}/${AUTHORIZED_KEYS}" chmod 600 "${JAILDIR}/${jail}/${AUTHORIZED_KEYS}" } set_ip() { jail="${1}" ip="${2}" if [ "${ip}" = "all" ] || [ "${ip}" = "0.0.0.0/0" ]; then ips="0.0.0.0/0" else ips=$(get_ip "${jail}") ips=$(echo "${ips}" "${ip}"|xargs -n1|grep -v "0.0.0.0/0"|sort|uniq) fi allow="AllowUsers" for ip in $ips; do allow="${allow} root@${ip}" done sed -i "s~^AllowUsers .*~${allow}~" "${JAILDIR}/$jail/${SSHD_CONFIG}" set_firewall "${jail}" } set_firewall() { jail="${1}" if [ -n "${FIREWALL_RULES}" ]; then if [ -f "${FIREWALL_RULES}" ]; then sed -i "/#${jail}$/d" "${FIREWALL_RULES}" fi if ( check_jail "${jail}" ); then port=$(get_port "${jail}") for ip in $(get_ip "${jail}"); do echo "/sbin/iptables -A INPUT -p tcp --sport 1024: --dport ${port} -s ${ip} -j ACCEPT #${jail}" >> "${FIREWALL_RULES}" done if [ -f /etc/init.d/minifirewall ]; then /etc/init.d/minifirewall restart >/dev/null fi fi fi } ## mk_jail function : create or update a jail mk_jail() { jail="${1}" passwd="${TPLDIR}/passwd" shadow="${TPLDIR}/shadow" group="${TPLDIR}/group" sshrc="${TPLDIR}/sshrc" [ -f "${LOCALTPLDIR}/passwd" ] && passwd="${LOCALTPLDIR}/passwd" [ -f "${LOCALTPLDIR}/shadow" ] && shadow="${LOCALTPLDIR}/shadow" [ -f "${LOCALTPLDIR}/group" ] && group="${LOCALTPLDIR}/group" [ -f "${LOCALTPLDIR}/sshrc" ] && group="${LOCALTPLDIR}/sshrc" umask 077 info "1 - Creating the chroot" cd "${JAILDIR}/${jail}" rm -rf bin lib lib64 run usr var/run etc/ssh/*key mkdir -p dev proc mkdir -p usr/bin usr/sbin usr/lib usr/lib/x86_64-linux-gnu usr/lib/openssh usr/lib64 mkdir -p etc/ssh var/log run/sshd mkdir -p root/.ssh var/backup -m 0700 ln -s usr/bin bin ln -s usr/lib lib ln -s usr/lib64 lib64 ln -st var ../run touch var/log/lastlog var/log/wtmp run/utmp info "2 - Copying essential files" [ -f /etc/ssh/ssh_host_rsa_key ] && cp /etc/ssh/ssh_host_rsa_key etc/ssh [ -f /etc/ssh/ssh_host_ecdsa_key ] && cp /etc/ssh/ssh_host_ecdsa_key etc/ssh [ -f /etc/ssh/ssh_host_ed25519_key ] && cp /etc/ssh/ssh_host_ed25519_key etc/ssh cp "${passwd}" etc cp "${shadow}" etc cp "${group}" etc cp "${sshrc}" etc/ssh info "3 - Copying binaries" cp -f /lib/ld-linux.so.2 lib 2>/dev/null || cp -f /lib64/ld-linux-x86-64.so.2 lib64 cp /lib/x86_64-linux-gnu/libnss* lib/x86_64-linux-gnu for dbin in /bin/sh /bin/ls /bin/mkdir /bin/cat /bin/rm /bin/sed /usr/bin/rsync /usr/bin/lastlog /usr/bin/touch /usr/sbin/sshd /usr/lib/openssh/sftp-server; do cp -f "${dbin}" "${JAILDIR}/${jail}/${dbin}"; for lib in $(ldd "${dbin}" | grep -Eo "/.*so.[0-9\.]+"); do cp -p "${lib}" "${JAILDIR}/${jail}/${lib}" done done } ## sub functions : functions call by subcommand sub_init() { jail="${1}" sshd_config="${TPLDIR}/sshd_config" inctpl="${TPLDIR}/inc.tpl" [ -f "${LOCALTPLDIR}/sshd_config" ] && sshd_config="${LOCALTPLDIR}/sshd_config" [ -f "${LOCALTPLDIR}/inc.tpl" ] && inctpl="${LOCALTPLDIR}/inc.tpl" check_jail "${jail}" && error "${jail} : trying to create existant jail" rootdir=$(dirname "${JAILDIR}") rootdir_inode=$(stat --format=%i "${rootdir}") jaildir_inode=$(stat --format=%i "${JAILDIR}") if [ "${rootdir_inode}" -eq 256 ] || [ "${jaildir_inode}" -eq 256 ]; then /bin/btrfs subvolume create "${JAILDIR}/${jail}" else mkdir -p "${JAILDIR}/${jail}" fi mk_jail "${jail}" info "4 - Copie default sshd_config" install -m 0640 "${sshd_config}" "${JAILDIR}/${jail}/${SSHD_CONFIG}" info "5 - Set usable sshd port" set_port "${jail}" auto info "6 - Copie default inc configuration" install -m 0640 "${inctpl}" "${CONFDIR}/${jail}" notice "${jail} : created jail" } sub_update() { jail="${1}" check_jail "${jail}" || error "${jail} : trying to update inexistant jail" check_jail_on "${jail}" && sub_stop "${jail}" mk_jail "${jail}" notice "${jail} : updated jail" } sub_remove() { jail="${1}" check_jail "${jail}" || error "${jail} : trying to remove inexistant jail" check_jail_on "${jail}" && sub_stop "${jail}" rm -f "${CONFDIR}/${jail}" jail_inode=$(stat --format=%i "${JAILDIR}/${jail}") if [ "${jail_inode}" -eq 256 ]; then /bin/btrfs subvolume delete "${JAILDIR}/${jail}" | debug else rm -rf "${JAILDIR}/${jail}" | debug fi if [ -d "${INCDIR}/${jail}" ]; then incs=$(ls "${INCDIR}/${jail}") for inc in ${incs}; do inc_inode=$(stat --format=%i "${INCDIR}/${jail}/${inc}") if [ "${inc_inode}" -eq 256 ]; then /bin/btrfs subvolume delete "${INCDIR}/${jail}/${inc}" | debug else warning "You need to purge ${INCDIR}/${jail}/${inc} manually !" fi done rmdir --ignore-fail-on-non-empty "${INCDIR}/${jail}" | debug fi set_firewall "${jail}" notice "${jail} : deleted jail" } sub_start() { jail="${1}" check_jail "${jail}" || error "${jail} : trying to start inexistant jail" check_jail_on "${jail}" && error "${jail} : trying to start already running jail" cd "${JAILDIR}/${jail}" grep -q "${JAILDIR}/${jail}/proc" /proc/mounts || mount -t proc "proc-${jail}" proc grep -q "${JAILDIR}/${jail}/dev" /proc/mounts || mount -nt tmpfs "dev-${jail}" dev [ -e "dev/console" ] || mknod -m 622 dev/console c 5 1 [ -e "dev/null" ] || mknod -m 666 dev/null c 1 3 [ -e "dev/zero" ] || mknod -m 666 dev/zero c 1 5 [ -e "dev/ptmx" ] || mknod -m 666 dev/ptmx c 5 2 [ -e "dev/tty" ] || mknod -m 666 dev/tty c 5 0 [ -e "dev/random" ] || mknod -m 444 dev/random c 1 8 [ -e "dev/urandom" ] || mknod -m 444 dev/urandom c 1 9 chown root:tty dev/console dev/ptmx dev/tty ln -fs proc/self/fd dev/fd ln -fs proc/self/fd/0 dev/stdin ln -fs proc/self/fd/1 dev/stdout ln -fs proc/self/fd/2 dev/stderr ln -fs proc/kcore dev/core mkdir -p dev/pts mkdir -p dev/shm grep -q "${JAILDIR}/${jail}/dev/pts" /proc/mounts || mount -t devpts -o gid=4,mode=620 none dev/pts grep -q "${JAILDIR}/${jail}/dev/shm" /proc/mounts || mount -t tmpfs none dev/shm chroot "${JAILDIR}/${jail}" /usr/sbin/sshd -E /var/log/authlog || error "${jail} : error on starting sshd" pid=$(cat "${JAILDIR}/${jail}/${SSHD_PID}") notice "${jail} was started [${pid}]" } sub_stop() { jail="${1}" check_jail "${jail}" || error "${jail} : trying to stop inexistant jail" check_jail_on "${jail}" || error "${jail} : trying to stop not running jail" pid=$(cat "${JAILDIR}/${jail}/${SSHD_PID}") for conn in $(ps --ppid "${pid}" -o pid=); do kill "${conn}" done kill "${pid}" && notice "${jail} was stopped [${pid}]" umount --lazy --recursive "${JAILDIR}/${jail}/dev" umount --lazy "${JAILDIR}/${jail}/proc/" } sub_reload() { jail="${1}" check_jail "${jail}" || error "${jail} : trying to reload inexistant jail" check_jail_on "${jail}" || error "${jail} : trying to reload not running jail" pid=$(cat "${JAILDIR}/${jail}/${SSHD_PID}") kill -HUP "${pid}" && notice "${jail} was reloaded [${pid}]" } sub_status() { jail="${1}" check_jail "${jail}" || error "${jail} : inexistant jail ! Use '$0 status' for list all" inc=$(get_inc "${jail}") if ( check_jail_on "${jail}" ); then status="ON " else status="OFF" fi port=$(get_port "${jail}") ip=$(get_ip "${jail}"|xargs|tr -s ' ' ',') echo "${jail} ${status} ${port} ${inc} ${ip}" | awk '{ printf("%- 30s %- 10s %- 10s %- 10s %- 40s\n", $1, $2, $3, $4, $5); }' } sub_params() { jail="${1}" params="${2}" option="${3}" check_jail "${jail}" || error "${jail} : inexistant jail'" if [ -z "${option}" ]; then "get_${params}" "${jail}" else "set_${params}" "${jail}" "${option}" check_jail_on "${jail}" && sub_reload "${jail}" notice "${jail} : update ${params} => ${option}" fi } sub_sync() { jail="${1}" check_jail "${jail}" || error "${jail} : trying to sync inexistant jail" [ -n "${NODE}" ] || error "Sync need config of \$NODE in /etc/default/bkctld !" jail="${1}" ssh "${NODE}" bkctld init "${jail}" | debug rsync -a "${JAILDIR}/${jail}/" "${NODE}:${JAILDIR}/${jail}/" --exclude proc/* --exclude sys/* --exclude dev/* --exclude run --exclude var/backup/* rsync -a "${CONFDIR}/${jail}" "${NODE}:${CONFDIR}/${jail}" if ( check_jail_on "${jail}" ); then ssh "${NODE}" bkctld start "${jail}" | debug fi if [ -n "${FIREWALL_RULES}" ]; then rsync -a "${FIREWALL_RULES}" "${NODE}:${FIREWALL_RULES}" ssh "${NODE}" /etc/init.d/minifirewall restart | debug fi } sub_inc() { date=$(date +"%Y-%m-%d-%H") jails=$(ls "${JAILDIR}") for jail in ${jails}; do inc="${INCDIR}/${jail}/${date}" mkdir -p "${INCDIR}/${jail}" if [ ! -d "${inc}" ]; then start=$(date +"%H:%M:%S") jail_inode=$(stat --format=%i "${JAILDIR}/${jail}") if [ "$jail_inode" -eq 256 ]; then /bin/btrfs subvolume snapshot -r "${JAILDIR}/${jail}" "${inc}" | debug else cp -alx "${JAILDIR}/${jail}/" "${inc}" | debug fi end=$(date +"%H:%M:%S") notice "${jail} : made ${date} inc [${start}/${end}]" else warning "${jail} : trying to made already existant inc" fi done } sub_rm() { empty="/tmp/bkctld-${$}-$(date +%N))" mkdir "${empty}" pidfile="/var/run/bkctld-rm.pid" if [ -f "${pidfile}" ]; then pid=$(cat "${pidfile}") ps -u "${pid}" >/dev/null if [ "${?}" -eq 0 ]; then kill -9 "${pid}" warning "${0} rm always run (PID ${pid}), killed by ${$} !" fi rm "${pidfile}" fi echo "${$}" > "${pidfile}" jails=$(ls "${JAILDIR}") for jail in ${jails}; do incs=$(ls "${INCDIR}/${jail}") if [ -f "${CONFDIR}/${jail}" ]; then keepfile="${CONFDIR}/.keep-${jail}" while read j; do date=$( echo "${j}" | cut -d. -f1 ) before=$( echo "${j}" | cut -d. -f2 ) date -d "$(date "${date}") ${before}" "+%Y-%m-%d" done < "${CONFDIR}/${jail}" > "${keepfile}" for j in $(echo "${incs}" | grep -v -f "${keepfile}"); do start=$(date +"%H:%M:%S") inc_inode=$(stat --format=%i "${INCDIR}/${jail}/${j}") if [ "${inc_inode}" -eq 256 ]; then /bin/btrfs subvolume delete "${INCDIR}/${jail}/${j}" | debug else cd "${INCDIR}/${jail}" rsync -a --delete "${empty}/" "${j}/" rmdir "${j}" fi end=$(date +"%H:%M:%S") notice "${jail} : deleted ${j} inc [${start}/${end}]" done fi done rmdir "${empty}" rm "${pidfile}" } sub_check() { cur_time=$(date "+%s") return=0 nb_crit=0 nb_warn=0 nb_ok=0 nb_unkn=0 output="" if [ -b "${BACKUP_DISK}" ]; then cryptsetup isLuks "${BACKUP_DISK}" if [ "$?" -eq 0 ]; then if [ ! -b '/dev/mapper/backup' ]; then echo "Luks disk ${BACKUP_DISK} is not mounted !\n" echo "cryptsetup luksOpen ${BACKUP_DISK} backup" exit 2 fi BACKUP_DISK='/dev/mapper/backup' fi grep -qE "^${BACKUP_DISK} " /etc/mtab if [ "$?" -ne 0 ]; then echo "Backup disk ${BACKUP_DISK} is not mounted !\n" echo "mount ${BACKUP_DISK} /backup" exit 2 fi fi jails=$(ls "${JAILDIR}") for jail in ${jails}; do if [ -f "${JAILDIR}/${jail}/var/log/lastlog" ]; then last_conn=$(stat --format=%Y "${JAILDIR}/${jail}/var/log/lastlog") date_diff=$(( (cur_time - last_conn) / (60*60) )) if [ "${date_diff}" -gt "${CRITICAL}" ]; then nb_crit=$((nb_crit + 1)) output="${output}CRITICAL - ${jail} - ${date_diff} hours\n" [ "${return}" -le 2 ] && return=2 elif [ "${date_diff}" -gt "${WARNING}" ]; then nb_warn=$((nb_warn + 1)) output="${output}WARNING - ${jail} - ${date_diff} hours\n" [ "${return}" -le 1 ] && return=1 else nb_ok=$((nb_ok + 1)) output="${output}OK - ${jail} - ${date_diff} hours\n" fi else nb_unkn=$((nb_unkn + 1)) output="${output}UNKNOWN - ${jail} doesn't have lastlog !\n" [ "${return}" -le 3 ] && return=3 fi done [ "${return}" -ge 0 ] && header="OK" [ "${return}" -ge 1 ] && header="WARNING" [ "${return}" -ge 2 ] && header="CRITICAL" [ "${return}" -ge 3 ] && header="UNKNOW" printf "%s - %s UNK / %s CRIT / %s WARN / %s OK\n\n" "${header}" "${nb_unkn}" "${nb_crit}" "${nb_warn}" "${nb_ok}" printf "${output}" | grep -E "^UNKNOW" printf "${output}" | grep -E "^CRITICAL" printf "${output}" | grep -E "^WARNING" printf "${output}" | grep -E "^OK" exit "${return}" } sub_stats() { lsof "${IDX_FILE}" >/dev/null 2>&1 || nohup sh -s -- </dev/null 2>&1 & ionice -c3 "${DUC}" index -d "${IDX_FILE}" "${JAILDIR}" touch "${INDEX_DIR}/.lastrun.duc" EOF [ ! -f "${INDEX_DIR}/.lastrun.duc" ] && notice "First run of DUC always in progress ..." && exit 0 [ ! -f ${IDX_FILE} ] && error "Index file do not exits !" printf "Last update of index file : " stat --format=%Y "${INDEX_DIR}/.lastrun.duc" | xargs -i -n1 date -R -d "@{}" echo " " | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $1, $2, $3, $4); }' duc_output=$(mktemp) stat_output=$(mktemp) incs_output=$(mktemp) trap "rm ${duc_output} ${incs_output} ${stat_output}" 0 "${DUC}" ls -d "${IDX_FILE}" "${JAILDIR}" > "${duc_output}" awk '{ print $2 }' "${duc_output}" | while read jail; do stat --format=%Y "/backup/jails/${jail}/var/log/lastlog" | xargs -i -n1 date -d "@{}" "+%d-%m-%Y" >> "${stat_output}" get_inc "${jail}" >> "${incs_output}" done paste "${duc_output}" "${incs_output}" "${stat_output}" | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $2, $1, $3, $4); }' } ## main function : check usage and valid params main() { [ "$(id -u)" -ne 0 ] && error "You need to be root to run ${0} !" [ -f /etc/default/bkctld ] && . /etc/default/bkctld CONFDIR="${CONFDIR:-/etc/evobackup}" BACKUP_DISK="${BACKUP_DISK:-}" JAILDIR="${JAILDIR:-/backup/jails}" INCDIR="${INCDIR:-/backup/incs}" TPLDIR="${TPLDIR:-/usr/share/bkctld}" INDEX_DIR="${INDEX_DIR:-/backup/index}" IDX_FILE="${IDX_FILE:-${INDEX_DIR}/bkctld-jails.idx}" LOCALTPLDIR="${LOCALTPLDIR:-/usr/local/share/bkctld}" SSHD_PID="${SSHD_PID:-/run/sshd.pid}" SSHD_CONFIG="${SSHD_CONFIG:-/etc/ssh/sshd_config}" AUTHORIZED_KEYS="${AUTHORIZED_KEYS:-/root/.ssh/authorized_keys}" FIREWALL_RULES="${FIREWALL_RULES:-}" LOGLEVEL="${LOGLEVEL:-6}" CRITICAL="${CRITICAL:-48}" WARNING="${WARNING:-24}" DUC=$(command -v duc-nox||command -v duc) mkdir -p "${CONFDIR}" "${JAILDIR}" "${INCDIR}" "${INDEX_DIR}" subcommand="${1:-}" jail="${2:-}" option="${3:-}" case "${subcommand}" in "" | "-h" | "--help") usage ;; "inc" | "rm" | "check" | "stats") "sub_${subcommand}" ;; "init") if [ -n "${jail}" ]; then "sub_${subcommand}" "${jail}" else usage fi ;; "key" | "port" | "ip") if [ -n "${jail}" ]; then sub_params "${jail}" "${subcommand}" "${option}" else usage fi ;; "start" | "stop" | "reload" | "restart" | "sync" | "update" | "remove") if [ -n "${jail}" ]; then if [ "${jail}" = "all" ]; then jails=$(ls "${JAILDIR}") for jail in ${jails}; do case "${subcommand}" in "start") check_jail_on "${jail}" || "sub_${subcommand}" "${jail}" ;; "stop" | "reload") check_jail_on "${jail}" && "sub_${subcommand}" "${jail}" ;; "restart") check_jail_on "${jail}" && sub_stop "${jail}" sub_start "${jail}" ;; *) "sub_${subcommand}" "${jail}" ;; esac done else if [ "${subcommand}" != "restart" ]; then "sub_${subcommand}" "${jail}" else check_jail_on "${jail}" && sub_stop "${jail}" sub_start "${jail}" fi fi else usage fi ;; "status") if [ -z "${jail}" ]; then jails=$(ls "${JAILDIR}") for jail in ${jails}; do "sub_${subcommand}" "${jail}" done else "sub_${subcommand}" "${jail}" fi ;; *) shift warning "'${subcommand}' is not a known subcommand." && usage exit 1 ;; esac } main "${@}"