evobackup/bkctld

#!/bin/sh
#
# bkctld is a shell script to create and manage a backup server which will
# handle the backup of many servers (clients).
#
# Author: Victor Laborie <vlaborie@evolix.fr>
# Contributor: Benoît Série <bserie@evolix.fr>, Gregory Colpart <reg@evolix.fr>, Romain Dessort <rdessort@evolix.fr>, Tristan Pilat <tpilat@evolix.fr>
# Licence: AGPLv3
#

set -u

usage(){
    cat <<EOF
Usage: $0 <subcommand> [options]
Subcommands:
    init        <jailname>                      Init jail <jailname>
    update      <jailname>|all                  Update jail <jailname> or all
    remove      <jailname>|all                  Remove jail <jailname> or all
    start       <jailname>|all                  Start jail <jailname> or all
    stop        <jailname>|all                  Stop jail <jailname> or all
    reload      <jailname>|all                  Reload jail <jailname> or all
    restart     <jailname>|all                  Restart jail <jailname> or all
    sync        <jailname>|all                  Sync jail <jailname> or all to another node
    status      [<jailname>]                    Print status of <jailname> (default all jail)
    key         <jailname>      [<keyfile>]     Set or get ssh pubic key of <jailname>
    port        <jailname>      [<port>|auto]   Set or get ssh port of <jailname>
    ip          <jailname>      [<ip>|all]      Set or get allowed(s) ip(s) of <jailname>
    inc                                         Make incremental inc of all jails
    rm                                          Remove old incremtal inc of all jails
    check                                       Run check on jails (NRPE output)
    stats                                       Make and display stats on jails (size, lastconn)

EOF
}

## logging functions

debug() {
    msg="${1:-$(cat /dev/stdin)}"
    if [ "${LOGLEVEL}" -ge 7 ]; then
        echo "${msg}"
        logger -t bkctld -p daemon.debug "${msg}"
    fi
}

info() {
    msg="${1:-$(cat /dev/stdin)}"
    if [ "${LOGLEVEL}" -ge 6 ]; then
        tty -s && echo "${msg}"
        logger -t bkctld -p daemon.info "${msg}"
    fi
}

notice() {
    msg="${1:-$(cat /dev/stdin)}"
    tty -s && echo "${msg}"
    [ "${LOGLEVEL}" -ge 5 ] && logger -t bkctld -p daemon.notice "${msg}"
}

warning() {
    msg="${1:-$(cat /dev/stdin)}"
    tty -s && echo "WARNING : ${msg}" >&2
    if [ "${LOGLEVEL}" -ge 4 ]; then
        tty -s || echo "WARNING : ${msg}" >&2
        logger -t bkctld -p daemon.warning "${msg}"
    fi
}

error() {
    msg="${1:-$(cat /dev/stdin)}"
    tty -s && echo "ERROR : ${msg}" >&2
    if [ "${LOGLEVEL}" -ge 5 ]; then
        tty -s || echo "ERROR : ${msg}" >&2
        logger -t bkctld -p daemon.error "${msg}"
    fi
    exit 1
}

## check functions

check_jail() {
    jail="${1}"
    [ -d "${JAILDIR}/${jail}" ] && return 0
    return 1
}

check_jail_on() {
    jail="${1}"
    return=1
    if [ -f "${JAILDIR}/${jail}/${SSHD_PID}" ]; then
        pid=$(cat "${JAILDIR}/${jail}/${SSHD_PID}")
        ps -p "${pid}" > /dev/null && return=0
    fi
    if [ "${return}" -eq 1 ]; then
        rm -f "${JAILDIR}/${jail}/${SSHD_PID}"
        grep -q "${JAILDIR}/${jail}/proc" /proc/mounts && umount --lazy "${JAILDIR}/${jail}/proc/"
        grep -q "${JAILDIR}/${jail}/dev" /proc/mounts && umount --lazy --recursive "${JAILDIR}/${jail}/dev"
    fi
    return "${return}"
}

## get functions : get info on jail

get_port() {
    jail="${1}"
    port=$(grep -E "Port [0-9]+" "${JAILDIR}/${jail}/${SSHD_CONFIG}"|grep -oE "[0-9]+")
    echo "${port}"
}

get_key() {
    jail="${1}"
    if [ -f "${JAILDIR}/${jail}/${AUTHORIZED_KEYS}" ]; then
        cat "${JAILDIR}/${jail}/${AUTHORIZED_KEYS}"
    fi
}

get_ip() {
    jail="${1}"
    grep -E "^AllowUsers" "${JAILDIR}/$jail/${SSHD_CONFIG}"|grep -Eo "root@[^ ]+"| while read allow; do
        echo "${allow}"|cut -d'@' -f2
    done
}

get_inc() {
    jail="${1}"
    inc="0"
        if [ -f "${CONFDIR}/${jail}" ]; then
            day=$(grep -c "day" "${CONFDIR}/${jail}")
            month=$(grep -c "month" "${CONFDIR}/${jail}")
            inc="${day}/${month}"
        fi
    echo "${inc}"
}

## set functions : set info on jail

set_port() {
    jail="${1}"
    port="${2}"
    if [ "${port}" = "auto" ]; then
        port=$(grep -h Port "${JAILDIR}"/*/"${SSHD_CONFIG}" 2>/dev/null | grep -Eo "[0-9]+" | sort -n | tail -1)
        port=$((port+1))
        [ "${port}" -le 1 ] && port=2222
    fi
    sed -i "s/^Port .*/Port ${port}/" "${JAILDIR}/$jail/${SSHD_CONFIG}"
    set_firewall "${jail}"
}

set_key() {
    jail="${1}"
    keyfile="${2}"
    [ -e "${keyfile}" ] || error "Keyfile ${keyfile} dosen't exist !"
    cat "${keyfile}" > "${JAILDIR}/${jail}/${AUTHORIZED_KEYS}"
    chmod 600 "${JAILDIR}/${jail}/${AUTHORIZED_KEYS}"
}

set_ip() {
    jail="${1}"
    ip="${2}"
    if [ "${ip}" = "all" ] || [ "${ip}" = "0.0.0.0/0" ]; then
        ips="0.0.0.0/0"
    else
        ips=$(get_ip "${jail}")
        ips=$(echo "${ips}" "${ip}"|xargs -n1|grep -v "0.0.0.0/0"|sort|uniq)
    fi
    allow="AllowUsers"
    for ip in $ips; do
        allow="${allow} root@${ip}"
    done
    sed -i "s~^AllowUsers .*~${allow}~" "${JAILDIR}/$jail/${SSHD_CONFIG}"
    set_firewall "${jail}"
}

set_firewall() {
    jail="${1}"
    if [ -n "${FIREWALL_RULES}" ]; then
        if [ -f "${FIREWALL_RULES}" ]; then
            sed -i "/#${jail}$/d" "${FIREWALL_RULES}"
        fi
        if ( check_jail "${jail}" ); then
            port=$(get_port "${jail}")
            for ip in $(get_ip "${jail}"); do
                echo "/sbin/iptables -A INPUT -p tcp --sport 1024: --dport ${port} -s ${ip} -j ACCEPT #${jail}" >> "${FIREWALL_RULES}"
            done
            if [ -f /etc/init.d/minifirewall ]; then
                /etc/init.d/minifirewall restart >/dev/null
            fi
        fi
    fi
}

## mk_jail function : create or update a jail

mk_jail() {
    jail="${1}"
    passwd="${TPLDIR}/passwd"
    shadow="${TPLDIR}/shadow"
    group="${TPLDIR}/group"
    sshrc="${TPLDIR}/sshrc"
    [ -f "${LOCALTPLDIR}/passwd" ] && passwd="${LOCALTPLDIR}/passwd"
    [ -f "${LOCALTPLDIR}/shadow" ] && shadow="${LOCALTPLDIR}/shadow"
    [ -f "${LOCALTPLDIR}/group" ] && group="${LOCALTPLDIR}/group"
    [ -f "${LOCALTPLDIR}/sshrc" ] && group="${LOCALTPLDIR}/sshrc"
    umask 077

    info "1 - Creating the chroot"
    cd "${JAILDIR}/${jail}"
    rm -rf bin lib lib64 run usr var/run etc/ssh/*key
    mkdir -p dev proc
    mkdir -p usr/bin usr/sbin usr/lib usr/lib/x86_64-linux-gnu usr/lib/openssh usr/lib64
    mkdir -p etc/ssh var/log run/sshd
    mkdir -p root/.ssh var/backup -m 0700
    ln -s usr/bin bin
    ln -s usr/lib lib
    ln -s usr/lib64 lib64
    ln -st var ../run
    touch var/log/lastlog var/log/wtmp run/utmp

    info "2 - Copying essential files"
    [ -f /etc/ssh/ssh_host_rsa_key ] && cp /etc/ssh/ssh_host_rsa_key etc/ssh
    [ -f /etc/ssh/ssh_host_ecdsa_key ] && cp /etc/ssh/ssh_host_ecdsa_key etc/ssh
    [ -f /etc/ssh/ssh_host_ed25519_key ] && cp /etc/ssh/ssh_host_ed25519_key etc/ssh
    cp "${passwd}" etc
    cp "${shadow}" etc
    cp "${group}" etc
    cp "${sshrc}" etc/ssh

    info "3 - Copying binaries"
    cp -f /lib/ld-linux.so.2 lib 2>/dev/null || cp -f /lib64/ld-linux-x86-64.so.2 lib64
    cp /lib/x86_64-linux-gnu/libnss* lib/x86_64-linux-gnu

    for dbin in /bin/sh /bin/ls /bin/mkdir /bin/cat /bin/rm /bin/sed /usr/bin/rsync /usr/bin/lastlog /usr/bin/touch /usr/sbin/sshd /usr/lib/openssh/sftp-server; do
        cp -f "${dbin}" "${JAILDIR}/${jail}/${dbin}";
        for lib in $(ldd "${dbin}" | grep -Eo "/.*so.[0-9\.]+"); do
            cp -p "${lib}" "${JAILDIR}/${jail}/${lib}"
        done
    done
}

## sub functions : functions call by subcommand
  
sub_init() {
    jail="${1}"
    sshd_config="${TPLDIR}/sshd_config"
    inctpl="${TPLDIR}/inc.tpl"
    [ -f "${LOCALTPLDIR}/sshd_config" ] && sshd_config="${LOCALTPLDIR}/sshd_config"
    [ -f "${LOCALTPLDIR}/inc.tpl" ] && inctpl="${LOCALTPLDIR}/inc.tpl"
    check_jail "${jail}" && error "${jail} : trying to create existant jail"

    rootdir=$(dirname "${JAILDIR}")
    rootdir_inode=$(stat --format=%i "${rootdir}")
    jaildir_inode=$(stat --format=%i "${JAILDIR}")
    if [ "${rootdir_inode}" -eq 256 ] || [ "${jaildir_inode}" -eq 256 ]; then
        /bin/btrfs subvolume create "${JAILDIR}/${jail}"
    else
        mkdir -p "${JAILDIR}/${jail}"
    fi
    mk_jail "${jail}"
    info "4 - Copie default sshd_config"
    install -m 0640 "${sshd_config}" "${JAILDIR}/${jail}/${SSHD_CONFIG}"
    info "5 - Set usable sshd port"
    set_port "${jail}" auto
    info "6 - Copie default inc configuration"
    install -m 0640 "${inctpl}" "${CONFDIR}/${jail}"
    notice "${jail} : created jail"
}

sub_update() {
    jail="${1}"
    check_jail "${jail}" || error "${jail} : trying to update inexistant jail"
    check_jail_on "${jail}" && sub_stop "${jail}"

    mk_jail "${jail}"
    notice "${jail} : updated jail"
}

sub_remove() {
    jail="${1}"
    check_jail "${jail}" || error "${jail} : trying to remove inexistant jail"

    check_jail_on "${jail}" && sub_stop "${jail}"
    
    rm -f "${CONFDIR}/${jail}"
    jail_inode=$(stat --format=%i "${JAILDIR}/${jail}")
    if [ "${jail_inode}" -eq 256 ]; then
        /bin/btrfs subvolume delete "${JAILDIR}/${jail}" | debug
    else
        rm -rf "${JAILDIR}/${jail}" | debug
    fi
    if [ -d  "${INCDIR}/${jail}" ]; then
        incs=$(ls "${INCDIR}/${jail}")
        for inc in ${incs}; do
            inc_inode=$(stat --format=%i "${INCDIR}/${jail}/${inc}")
            if [ "${inc_inode}" -eq 256 ]; then
                /bin/btrfs subvolume delete "${INCDIR}/${jail}/${inc}" | debug
            else
                warning "You need to purge ${INCDIR}/${jail}/${inc} manually !"
            fi
        done
        rmdir --ignore-fail-on-non-empty "${INCDIR}/${jail}" | debug
    fi
    set_firewall "${jail}"
    notice "${jail} : deleted jail"
}

sub_start() {
    jail="${1}"
    check_jail "${jail}" || error "${jail} : trying to start inexistant jail"
    check_jail_on "${jail}" && error "${jail} : trying to start already running jail"

    cd "${JAILDIR}/${jail}"
    grep -q "${JAILDIR}/${jail}/proc" /proc/mounts || mount -t proc "proc-${jail}" proc
    grep -q "${JAILDIR}/${jail}/dev" /proc/mounts || mount -nt tmpfs "dev-${jail}" dev
    [ -e "dev/console" ] || mknod -m 622 dev/console c 5 1
    [ -e "dev/null" ] || mknod -m 666 dev/null c 1 3
    [ -e "dev/zero" ] || mknod -m 666 dev/zero c 1 5
    [ -e "dev/ptmx" ] || mknod -m 666 dev/ptmx c 5 2
    [ -e "dev/tty" ] || mknod -m 666 dev/tty c 5 0
    [ -e "dev/random" ] || mknod -m 444 dev/random c 1 8
    [ -e "dev/urandom" ] || mknod -m 444 dev/urandom c 1 9
    chown root:tty dev/console dev/ptmx dev/tty
    ln -fs proc/self/fd dev/fd
    ln -fs proc/self/fd/0 dev/stdin
    ln -fs proc/self/fd/1 dev/stdout
    ln -fs proc/self/fd/2 dev/stderr
    ln -fs proc/kcore dev/core
    mkdir -p dev/pts
    mkdir -p dev/shm
    grep -q "${JAILDIR}/${jail}/dev/pts" /proc/mounts || mount -t devpts -o gid=4,mode=620 none dev/pts
    grep -q "${JAILDIR}/${jail}/dev/shm" /proc/mounts || mount -t tmpfs none dev/shm
    chroot "${JAILDIR}/${jail}" /usr/sbin/sshd -E /var/log/authlog || error "${jail} : error on starting sshd"
    pid=$(cat "${JAILDIR}/${jail}/${SSHD_PID}")
    notice "${jail} was started [${pid}]"
}

sub_stop() {
    jail="${1}"
    check_jail "${jail}" || error "${jail} : trying to stop inexistant jail"
    check_jail_on "${jail}" || error "${jail} : trying to stop not running jail"

    pid=$(cat "${JAILDIR}/${jail}/${SSHD_PID}")
    for conn in $(ps --ppid "${pid}" -o pid=); do
        kill "${conn}"
    done
    kill "${pid}" && notice "${jail} was stopped [${pid}]"
    umount --lazy --recursive "${JAILDIR}/${jail}/dev"
    umount --lazy "${JAILDIR}/${jail}/proc/"
}

sub_reload() {
    jail="${1}"
    check_jail "${jail}" || error "${jail} : trying to reload inexistant jail"
    check_jail_on "${jail}" || error "${jail} : trying to reload not running jail"

    pid=$(cat "${JAILDIR}/${jail}/${SSHD_PID}")

    kill -HUP "${pid}" && notice "${jail} was reloaded [${pid}]"
}

sub_status() {
    jail="${1}"
    check_jail "${jail}" || error "${jail} : inexistant jail ! Use '$0 status' for list all"

    inc=$(get_inc "${jail}")
    if ( check_jail_on "${jail}" ); then
        status="ON "
    else
        status="OFF"
    fi
    port=$(get_port "${jail}")
    ip=$(get_ip "${jail}"|xargs|tr -s ' ' ',')
    echo "${jail} ${status} ${port} ${inc} ${ip}" | awk '{ printf("%- 30s %- 10s %- 10s %- 10s %- 40s\n", $1, $2, $3, $4, $5); }'
}

sub_params() {
    jail="${1}"
    params="${2}"
    option="${3}"
    check_jail "${jail}" || error "${jail} : inexistant jail'"

    if [ -z "${option}" ]; then
        "get_${params}" "${jail}"
    else
        "set_${params}" "${jail}" "${option}"
        check_jail_on "${jail}" && sub_reload "${jail}"
        notice "${jail} : update ${params} => ${option}"
    fi
}

sub_sync() {
    jail="${1}"
    check_jail "${jail}" || error "${jail} : trying to sync inexistant jail"

    [ -n "${NODE}" ] || error "Sync need config of \$NODE in /etc/default/bkctld !"

    jail="${1}"
    ssh "${NODE}" bkctld init "${jail}" | debug
    rsync -a "${JAILDIR}/${jail}/" "${NODE}:${JAILDIR}/${jail}/" --exclude proc/* --exclude sys/* --exclude dev/* --exclude run --exclude var/backup/*
    rsync -a "${CONFDIR}/${jail}" "${NODE}:${CONFDIR}/${jail}"
    if ( check_jail_on "${jail}" ); then
        ssh "${NODE}" bkctld start "${jail}" | debug
    fi
    if [ -n "${FIREWALL_RULES}" ]; then
        rsync -a "${FIREWALL_RULES}" "${NODE}:${FIREWALL_RULES}"
        ssh "${NODE}" /etc/init.d/minifirewall restart | debug
    fi
}

sub_inc() {
    date=$(date +"%Y-%m-%d-%H")
    jails=$(ls "${JAILDIR}")
    for jail in ${jails}; do
        inc="${INCDIR}/${jail}/${date}"
        mkdir -p "${INCDIR}/${jail}"
        if [ ! -d "${inc}" ]; then
            start=$(date +"%H:%M:%S")
            jail_inode=$(stat --format=%i "${JAILDIR}/${jail}")
            if [ "$jail_inode" -eq 256 ]; then
                /bin/btrfs subvolume snapshot -r "${JAILDIR}/${jail}" "${inc}" | debug
            else
                cp -alx "${JAILDIR}/${jail}/" "${inc}" | debug
            fi
            end=$(date +"%H:%M:%S")
            notice "${jail} : made ${date} inc [${start}/${end}]"
        else
            warning "${jail} : trying to made already existant inc"
        fi
    done
}

sub_rm() {
    empty="/tmp/bkctld-${$}-$(date +%N))"
    mkdir "${empty}"
    pidfile="/var/run/bkctld-rm.pid"
    if [ -f "${pidfile}" ]; then
        pid=$(cat "${pidfile}")
        ps -u "${pid}" >/dev/null
        if [ "${?}" -eq 0 ]; then
            kill -9 "${pid}"
            warning "${0} rm always run (PID ${pid}), killed by ${$} !"
        fi
        rm "${pidfile}"
        fi
    echo "${$}" > "${pidfile}"
    jails=$(ls "${JAILDIR}")
    for jail in ${jails}; do
        incs=$(ls "${INCDIR}/${jail}")
        if [ -f "${CONFDIR}/${jail}" ]; then
            keepfile="${CONFDIR}/.keep-${jail}"
            while read j; do
                date=$( echo "${j}" | cut -d. -f1 )
                before=$( echo "${j}" | cut -d. -f2 )
                date -d "$(date "${date}") ${before}" "+%Y-%m-%d"
            done < "${CONFDIR}/${jail}" > "${keepfile}"
            for j in $(echo "${incs}" | grep -v -f "${keepfile}"); do
                start=$(date +"%H:%M:%S")
                inc_inode=$(stat --format=%i "${INCDIR}/${jail}/${j}")
                if [ "${inc_inode}" -eq 256 ]; then
                    /bin/btrfs subvolume delete "${INCDIR}/${jail}/${j}" | debug
                else
                    cd "${INCDIR}/${jail}"
                    rsync -a --delete "${empty}/" "${j}/"
                    rmdir "${j}"
                fi
                end=$(date +"%H:%M:%S")
                notice "${jail} : deleted ${j} inc [${start}/${end}]"
            done
        fi
    done
    rmdir "${empty}"
    rm "${pidfile}"
}

sub_check() {
    cur_time=$(date "+%s")
    return=0
    nb_crit=0
    nb_warn=0
    nb_ok=0
    nb_unkn=0
    output=""

    if [ -b "${BACKUP_DISK}" ]; then
	cryptsetup isLuks "${BACKUP_DISK}"
        if [ "$?" -eq 0 ]; then
            if [ ! -b '/dev/mapper/backup' ]; then
                echo "Luks disk ${BACKUP_DISK} is not mounted !\n"
                echo "cryptsetup luksOpen ${BACKUP_DISK} backup"
                exit 2
            fi
            BACKUP_DISK='/dev/mapper/backup'
        fi
        grep -qE "^${BACKUP_DISK} " /etc/mtab
        if [ "$?" -ne 0 ]; then
             echo "Backup disk ${BACKUP_DISK} is not mounted !\n"
             echo "mount ${BACKUP_DISK} /backup"
             exit 2
        fi
    fi

    jails=$(ls "${JAILDIR}")
    for jail in ${jails}; do
        if [ -f "${JAILDIR}/${jail}/var/log/lastlog" ]; then
            last_conn=$(stat --format=%Y "${JAILDIR}/${jail}/var/log/lastlog")
            date_diff=$(( (cur_time - last_conn) / (60*60) ))
            if [ "${date_diff}" -gt "${CRITICAL}" ]; then
                nb_crit=$((nb_crit + 1))
                output="${output}CRITICAL - ${jail} - ${date_diff} hours\n"
                [ "${return}" -le 2 ] && return=2
            elif [ "${date_diff}" -gt "${WARNING}" ]; then
                nb_warn=$((nb_warn + 1))
                output="${output}WARNING - ${jail} - ${date_diff} hours\n"
                [ "${return}" -le 1 ] && return=1
            else
                nb_ok=$((nb_ok + 1))
                output="${output}OK - ${jail} - ${date_diff} hours\n"
            fi
        else
            nb_unkn=$((nb_unkn + 1))
            output="${output}UNKNOWN - ${jail} doesn't have lastlog !\n"
            [ "${return}" -le 3 ] && return=3
        fi
    done

    [ "${return}" -ge 0 ] && header="OK"
    [ "${return}" -ge 1 ] && header="WARNING"
    [ "${return}" -ge 2 ] && header="CRITICAL"
    [ "${return}" -ge 3 ] && header="UNKNOW"

    printf "%s - %s UNK / %s CRIT / %s WARN / %s OK\n\n" "${header}" "${nb_unkn}" "${nb_crit}" "${nb_warn}" "${nb_ok}"

    printf "${output}" | grep -E "^UNKNOW"
    printf "${output}" | grep -E "^CRITICAL"
    printf "${output}" | grep -E "^WARNING"
    printf "${output}" | grep -E "^OK"

    exit "${return}"
}

sub_stats() {
    lsof "${IDX_FILE}" >/dev/null 2>&1 || nohup sh -s -- <<EOF >/dev/null 2>&1 &
ionice -c3 "${DUC}" index -d "${IDX_FILE}" "${JAILDIR}"
touch "${INDEX_DIR}/.lastrun.duc"
EOF
    [ ! -f "${INDEX_DIR}/.lastrun.duc" ] && notice "First run of DUC always in progress ..." && exit 0
    [ ! -f ${IDX_FILE} ] && error "Index file do not exits !"
    printf "Last update of index file : "
    stat --format=%Y "${INDEX_DIR}/.lastrun.duc" | xargs -i -n1 date -R -d "@{}"
    echo "<jail> <size> <incs> <lastconn>" | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $1, $2, $3, $4); }'
    duc_output=$(mktemp)
    stat_output=$(mktemp)
    incs_output=$(mktemp)
    trap "rm ${duc_output} ${incs_output} ${stat_output}" 0
    "${DUC}" ls -d "${IDX_FILE}" "${JAILDIR}" > "${duc_output}"
    awk '{ print $2 }' "${duc_output}" | while read jail; do
        stat --format=%Y "/backup/jails/${jail}/var/log/lastlog" | xargs -i -n1 date -d "@{}" "+%d-%m-%Y" >> "${stat_output}"
        get_inc "${jail}" >> "${incs_output}"
    done
    paste "${duc_output}" "${incs_output}" "${stat_output}" | awk '{ printf("%- 30s %- 10s %- 10s %- 15s\n", $2, $1, $3, $4); }'
}

## main function : check usage and valid params

main() {
    [ "$(id -u)" -ne 0 ] && error "You need to be root to run ${0} !"

    [ -f /etc/default/bkctld ] && . /etc/default/bkctld
    CONFDIR="${CONFDIR:-/etc/evobackup}"
    BACKUP_DISK="${BACKUP_DISK:-}"
    JAILDIR="${JAILDIR:-/backup/jails}"
    INCDIR="${INCDIR:-/backup/incs}"
    TPLDIR="${TPLDIR:-/usr/share/bkctld}"
    INDEX_DIR="${INDEX_DIR:-/backup/index}"
    IDX_FILE="${IDX_FILE:-${INDEX_DIR}/bkctld-jails.idx}"
    LOCALTPLDIR="${LOCALTPLDIR:-/usr/local/share/bkctld}"
    SSHD_PID="${SSHD_PID:-/run/sshd.pid}"
    SSHD_CONFIG="${SSHD_CONFIG:-/etc/ssh/sshd_config}"
    AUTHORIZED_KEYS="${AUTHORIZED_KEYS:-/root/.ssh/authorized_keys}"
    FIREWALL_RULES="${FIREWALL_RULES:-}"
    LOGLEVEL="${LOGLEVEL:-6}"
    CRITICAL="${CRITICAL:-48}"
    WARNING="${WARNING:-24}"
    DUC=$(command -v duc-nox||command -v duc)
    mkdir -p "${CONFDIR}" "${JAILDIR}" "${INCDIR}" "${INDEX_DIR}"
    subcommand="${1:-}"
    jail="${2:-}"
    option="${3:-}"
    case "${subcommand}" in
        "" | "-h" | "--help")
            usage
            ;;
        "inc" | "rm" | "check" | "stats")
            "sub_${subcommand}"
        ;;
            "init")
        if [ -n "${jail}" ]; then
            "sub_${subcommand}" "${jail}"
        else
            usage
        fi
        ;;
        "key" | "port" | "ip")
        if [ -n "${jail}" ]; then
            sub_params "${jail}" "${subcommand}" "${option}"
        else
            usage
        fi
        ;;
        "start" | "stop" | "reload" | "restart" | "sync" | "update" | "remove")
        if [ -n "${jail}" ]; then
            if [ "${jail}" = "all" ]; then
                jails=$(ls "${JAILDIR}")
                for jail in ${jails}; do
                    case "${subcommand}" in
                        "start")
                            check_jail_on "${jail}" || "sub_${subcommand}" "${jail}"
                        ;;
                        "stop" | "reload")
                            check_jail_on "${jail}" && "sub_${subcommand}" "${jail}"
                        ;;
                        "restart")
                            check_jail_on "${jail}" && sub_stop "${jail}"
                            sub_start "${jail}"
                        ;;
                        *)
                            "sub_${subcommand}" "${jail}"
                        ;;
                    esac
                        done
            else
                if [ "${subcommand}" != "restart" ]; then
                    "sub_${subcommand}" "${jail}"
                else
                    check_jail_on "${jail}" && sub_stop "${jail}"
                    sub_start "${jail}"
                fi
            fi
        else
            usage
        fi
        ;;
        "status")
        if [ -z "${jail}" ]; then
            jails=$(ls "${JAILDIR}")
            for jail in ${jails}; do
                "sub_${subcommand}" "${jail}"
            done
        else
            "sub_${subcommand}" "${jail}"
        fi
        ;;
        *)
            shift
            warning "'${subcommand}' is not a known subcommand." && usage
            exit 1
            ;;
    esac
}

main "${@}"