908 lines
26 KiB
Bash
Executable file
908 lines
26 KiB
Bash
Executable file
#!/bin/bash
|
|
|
|
VERSION="24.03"
|
|
|
|
# Common functions for "repair" and "restart" scripts
|
|
|
|
set -u
|
|
|
|
# Initializes the program, context, configuration…
|
|
initialize() {
|
|
PATH="${PATH}":/usr/sbin:/sbin
|
|
|
|
# Used in many places to refer to the program name.
|
|
# Examples: repair_mysql, restart_nrpe…
|
|
PROGNAME=$(basename "${0}")
|
|
|
|
# find out if running in interactive mode, or not
|
|
if [ -t 0 ]; then
|
|
INTERACTIVE=1
|
|
else
|
|
INTERACTIVE=0
|
|
fi
|
|
readonly INTERACTIVE
|
|
|
|
# Default empty value for Debug mode
|
|
DEBUG="${DEBUG:-""}"
|
|
|
|
# Repair scripts obey to the value of a variable named after the script
|
|
# You can set the value ("on" or "off") in /etc/evolinux/autosysadmin
|
|
# Here we set the default value to "on".
|
|
declare -g "${PROGNAME}"=on # dynamic variable assignment ($PROGNAME == repair_*)
|
|
|
|
PID=$$
|
|
readonly PID
|
|
|
|
# Each execution (run) gets a unique ID
|
|
RUN_ID="$(date +"%Y-%m-%d_%H-%M")_${PROGNAME}_${PID}"
|
|
readonly RUN_ID
|
|
|
|
# Main log directory
|
|
MAIN_LOG_DIR="/var/log/autosysadmin"
|
|
readonly MAIN_LOG_DIR
|
|
# shellcheck disable=SC2174
|
|
mkdir --mode=750 --parents "${MAIN_LOG_DIR}"
|
|
chgrp adm "${MAIN_LOG_DIR}"
|
|
|
|
# Each execution store some information
|
|
# in a unique directory based on the RUN_ID
|
|
RUN_LOG_DIR="${MAIN_LOG_DIR}/${RUN_ID}"
|
|
readonly RUN_LOG_DIR
|
|
# shellcheck disable=SC2174
|
|
mkdir --mode=750 --parents "${RUN_LOG_DIR}"
|
|
chgrp adm "${RUN_LOG_DIR}"
|
|
|
|
# This log file contains all events
|
|
RUN_LOG_FILE="${RUN_LOG_DIR}/autosysadmin.log"
|
|
readonly RUN_LOG_FILE
|
|
|
|
# This log file contains notable actions
|
|
ACTIONS_FILE="${RUN_LOG_DIR}/actions.log"
|
|
readonly ACTIONS_FILE
|
|
touch "${ACTIONS_FILE}"
|
|
# This log file contains abort reasons (if any)
|
|
ABORT_FILE="${RUN_LOG_DIR}/abort.log"
|
|
readonly ABORT_FILE
|
|
# touch "${ABORT_FILE}"
|
|
|
|
# Date format for log messages
|
|
DATE_FORMAT="%Y-%m-%d %H:%M:%S"
|
|
|
|
# This will contain lock, last-run markers…
|
|
# It's ok to lose the content after a reboot
|
|
RUN_DIR="/run/autosysadmin"
|
|
readonly RUN_DIR
|
|
mkdir -p "${RUN_DIR}"
|
|
|
|
# Only a singe instace of each script can run simultaneously
|
|
# We use a customizable lock name for this.
|
|
# By default it's the script's name
|
|
LOCK_NAME=${LOCK_NAME:-${PROGNAME}}
|
|
# If a lock is found, we can wait for it to disappear.
|
|
# The value must be understood by sleep(1)
|
|
LOCK_WAIT="0"
|
|
|
|
# Default values for email headers
|
|
EMAIL_FROM="equipe+autosysadmin@evolix.fr"
|
|
EMAIL_INTERNAL="autosysadmin@evolix.fr"
|
|
|
|
LOCK_FILE="${RUN_DIR}/${LOCK_NAME}.lock"
|
|
readonly LOCK_FILE
|
|
# Remove lock file at exit
|
|
cleanup() {
|
|
# shellcheck disable=SC2317
|
|
rm -f "${LOCK_FILE}"
|
|
}
|
|
trap 'cleanup' 0
|
|
|
|
# Load configuration
|
|
# shellcheck disable=SC1091
|
|
test -f /etc/evolinux/autosysadmin && source /etc/evolinux/autosysadmin
|
|
|
|
log_all "Begin ${PROGNAME} RUN_ID: ${RUN_ID}"
|
|
log_all "Log directory is ${RUN_LOG_DIR}"
|
|
}
|
|
|
|
# Executes a list of tasks before exiting:
|
|
# * prepare a summary of actions and possible abort reasons
|
|
# * send emails
|
|
# * do some cleanup
|
|
quit() {
|
|
log_all "End ${PROGNAME} RUN_ID: ${RUN_ID}"
|
|
|
|
summary="RUN_ID: ${RUN_ID}"
|
|
if [ -s "${ABORT_FILE}" ]; then
|
|
# Add abort reasons to summary
|
|
summary="${summary}\n$(print_abort_reasons)"
|
|
hook_mail "abort"
|
|
|
|
return_code=1
|
|
else
|
|
if [ -s "${ACTIONS_FILE}" ]; then
|
|
# Add notable actions to summary
|
|
summary="${summary}\n$(print_actions "Aucune action")"
|
|
hook_mail "success"
|
|
fi
|
|
|
|
return_code=0
|
|
fi
|
|
|
|
hook_mail "internal"
|
|
|
|
if is_interactive; then
|
|
# shellcheck disable=SC2001
|
|
echo "${summary}" | sed -e 's/\\n/\n/g'
|
|
else
|
|
/usr/share/scripts/evomaintenance.sh --auto --user autosysadmin --message "${summary}" --no-commit --no-mail
|
|
fi
|
|
|
|
teardown
|
|
|
|
# shellcheck disable=SC2086
|
|
exit ${return_code}
|
|
}
|
|
|
|
teardown() {
|
|
:
|
|
}
|
|
|
|
# Return true/false
|
|
is_interactive() {
|
|
test "${INTERACTIVE}" -eq "1"
|
|
}
|
|
|
|
save_server_state() {
|
|
DUMP_SERVER_STATE_BIN="$(command -v dump-server-state || command -v backup-server-state)"
|
|
|
|
if [ -z "${DUMP_SERVER_STATE_BIN}" ]; then
|
|
log_all "Warning: dump-server-state is not present. No server state recorded."
|
|
fi
|
|
|
|
if [ -x "${DUMP_SERVER_STATE_BIN}" ]; then
|
|
DUMP_DIR=$(file_path_in_log_dir "server-state")
|
|
# We don't want the logging to take too much time,
|
|
# so we kill it if it takes more than 20 seconds.
|
|
timeout --signal 9 20 \
|
|
"${DUMP_SERVER_STATE_BIN}" \
|
|
--dump-dir="${DUMP_DIR}" \
|
|
--df \
|
|
--dmesg \
|
|
--iptables \
|
|
--lxc \
|
|
--netcfg \
|
|
--netstat \
|
|
--uname \
|
|
--processes \
|
|
--systemctl \
|
|
--uptime \
|
|
--virsh \
|
|
--disks \
|
|
--mysql-processes \
|
|
--no-apt-states \
|
|
--no-apt-config \
|
|
--no-dpkg-full \
|
|
--no-dpkg-status \
|
|
--no-mount \
|
|
--no-packages \
|
|
--no-sysctl \
|
|
--no-etc
|
|
|
|
log_run "Server state saved in \`server-state' directory."
|
|
fi
|
|
}
|
|
|
|
is_debug() {
|
|
# first time: do the check…
|
|
# other times: pass
|
|
if [ -z "${DEBUG:-""}" ]; then
|
|
debug_file="/etc/evolinux/autosysadmin.debug"
|
|
|
|
if [ -e "${debug_file}" ]; then
|
|
last_change=$(stat -c %Z "${debug_file}")
|
|
limit_date=$(date --date "14400 seconds ago" +"%s")
|
|
|
|
if [ $(( last_change - limit_date )) -le "0" ]; then
|
|
log_run "Debug mode disabled; file is too old (%{last_change} seconds)."
|
|
rm "${debug_file}"
|
|
# Debug mode disabled
|
|
DEBUG="0"
|
|
else
|
|
log_run "Debug mode enabled."
|
|
# Debug mode enabled
|
|
DEBUG="1"
|
|
fi
|
|
else
|
|
# log_run "Debug mode disabled; file is absent."
|
|
# Debug mode disabled
|
|
DEBUG="0"
|
|
fi
|
|
fi
|
|
# return the value
|
|
test "${DEBUG}" -eq "1"
|
|
}
|
|
|
|
# Uses the who(1) definition of "active"
|
|
currently_active_users() {
|
|
LC_ALL=C who --users | grep --extended-regexp "\s+\.\s+" | awk '{print $1}' | sort --human-numeric-sort | uniq
|
|
}
|
|
# Users active in the last 29 minutes
|
|
recently_active_users() {
|
|
LC_ALL=C who --users | grep --extended-regexp "\s+00:(0|1|2)[0-9]\s+" | awk --field-separator ' ' '{print $1,$6}'
|
|
}
|
|
# Save the list of users to a file in the log directory
|
|
save_active_users() {
|
|
LC_ALL=C who --users | save_in_log_dir "who-users"
|
|
}
|
|
|
|
# An autosysadmin must not perform actions if a user is active or was active recently.
|
|
#
|
|
# This can by bypassed in interactive mode.
|
|
# It's OK to lose this data after a reboot.
|
|
ensure_no_active_users_or_exit() {
|
|
# Save all active users
|
|
save_active_users
|
|
|
|
if is_debug; then
|
|
log_run "Debug mode enabled: continue without checking active users."
|
|
return 0;
|
|
fi
|
|
|
|
# Is there any currently active user?
|
|
currently_active_users=$(currently_active_users)
|
|
if [ -n "${currently_active_users}" ]; then
|
|
# shellcheck disable=SC2001
|
|
users_oneliner=$(echo "${currently_active_users}" | sed -e 's/\n/ /')
|
|
log_run "Currently active users: ${users_oneliner}"
|
|
if is_interactive; then
|
|
echo "Some users are currently active:"
|
|
# shellcheck disable=SC2001
|
|
echo "${currently_active_users}" | sed -e 's/\(.\+\)/* \1/'
|
|
answer=""
|
|
while :; do
|
|
printf "> Continue? [Y,n,?] "
|
|
read -r answer
|
|
case ${answer} in
|
|
[Yy]|"" )
|
|
log_run "Active users check bypassed manually in interactive mode."
|
|
return
|
|
;;
|
|
[Nn] )
|
|
log_run "Active users check confirmed manually in interactive mode."
|
|
log_abort_and_quit "Active users detected: ${users_oneliner}"
|
|
;;
|
|
* )
|
|
printf "y - yes, continue\n"
|
|
printf "n - no, exit\n"
|
|
printf "? - print this help\n"
|
|
;;
|
|
esac
|
|
done
|
|
else
|
|
log_abort_and_quit "Currently active users detected: ${users_oneliner}."
|
|
fi
|
|
else
|
|
# or recently (the last 30 minutes) active user?
|
|
recently_active_users=$(recently_active_users)
|
|
if [ -n "${recently_active_users}" ]; then
|
|
# shellcheck disable=SC2001
|
|
users_oneliner=$(echo "${recently_active_users}" | sed -e 's/\n/ /')
|
|
log_run "Recently active users: ${users_oneliner}"
|
|
if is_interactive; then
|
|
echo "Some users were recently active:"
|
|
# shellcheck disable=SC2001
|
|
echo "${recently_active_users}" | sed -e 's/\(.\+\)/* \1/'
|
|
answer=""
|
|
while :; do
|
|
printf "> Continue? [Y,n,?] "
|
|
read -r answer
|
|
case ${answer} in
|
|
[Yy]|"" )
|
|
log_run "Active users check bypassed manually in interactive mode."
|
|
return
|
|
;;
|
|
[Nn] )
|
|
log_run "Active users check confirmed manually in interactive mode."
|
|
log_abort_and_quit "Recently active users detected: ${users_oneliner}."
|
|
;;
|
|
* )
|
|
printf "y - yes, continue\n"
|
|
printf "n - no, exit\n"
|
|
printf "? - print this help\n"
|
|
;;
|
|
esac
|
|
done
|
|
else
|
|
log_abort_and_quit "Recently active users detected: ${users_oneliner}."
|
|
fi
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Takes an NRPE command name as 1st parameter,
|
|
# and executes the full command if found in the configuration.
|
|
# Return the result and the return code of the command.
|
|
check_nrpe() {
|
|
check="$1"
|
|
|
|
nrpe_files=""
|
|
|
|
# Check if NRPE config is found
|
|
if [ -f "/etc/nagios/nrpe.cfg" ]; then
|
|
nrpe_files="${nrpe_files} /etc/nagios/nrpe.cfg"
|
|
else
|
|
msg="NRPE configuration not found: /etc/nagios/nrpe.cfg"
|
|
log_run "${msg}"
|
|
echo "${msg}"
|
|
return 3
|
|
fi
|
|
|
|
# Search for included files
|
|
# shellcheck disable=SC2086
|
|
while IFS= read -r include_file; do
|
|
nrpe_files="${nrpe_files} ${include_file}"
|
|
done < <(grep --extended-regexp '^\s*include=.+' ${nrpe_files} | cut -d = -f 2)
|
|
|
|
# Search for files in included directories
|
|
# shellcheck disable=SC2086
|
|
while IFS= read -r include_dir; do
|
|
nrpe_files="${nrpe_files} ${include_dir}/*.cfg"
|
|
done < <(grep --extended-regexp '^\s*include_dir=.+' ${nrpe_files} | cut -d = -f 2)
|
|
|
|
# Fetch uncommented commands in (sorted) config files
|
|
# shellcheck disable=SC2086
|
|
nrpe_commands=$(grep --no-filename --exclude=*~ --fixed-strings "[${check}]" ${nrpe_files} | grep --invert-match --extended-regexp '^\s*#\s*command' | cut -d = -f 2)
|
|
nrpe_commands_count=$(echo "${nrpe_commands}" | wc -l)
|
|
|
|
if is_debian_version "9" "<=" && [ "${nrpe_commands_count}" -gt "1" ]; then
|
|
# On Debian <= 9, NRPE loading was not sorted
|
|
# we need to raise an error if we have multiple defined commands
|
|
msg="Unable to determine which NRPE command to run"
|
|
log_run "${msg}"
|
|
echo "${msg}"
|
|
return 3
|
|
else
|
|
# On Debian > 9, use the last command
|
|
nrpe_command=$(echo "${nrpe_commands}" | tail -n 1)
|
|
|
|
nrpe_result=$(${nrpe_command})
|
|
nrpe_rc=$?
|
|
|
|
log_run "NRPE command (exited with ${nrpe_rc}): ${nrpe_command}"
|
|
log_run "${nrpe_result}"
|
|
|
|
echo "${nrpe_result}"
|
|
return "${nrpe_rc}"
|
|
fi
|
|
}
|
|
|
|
# An autosysadmin script must not run twice (or more) simultaneously.
|
|
# We use a customizable (with LOCK_NAME) lock file to keep track of this.
|
|
# A wait time can be configured.
|
|
#
|
|
# This can by bypassed in interactive mode.
|
|
# It's OK to lose this data after a reboot.
|
|
acquire_lock_or_exit() {
|
|
lock_file="${1:-${LOCK_FILE}}"
|
|
lock_wait="${2:-${LOCK_WAIT}}"
|
|
|
|
# lock_wait must be compatible with sleep(1), otherwise fallback to 0
|
|
if ! echo "${lock_wait}" | grep -Eq '^[0-9]+[smhd]?$'; then
|
|
log_run "Lock wait: incorrect value '${lock_wait}', fallback to 0."
|
|
lock_wait=0
|
|
fi
|
|
|
|
if [ "${lock_wait}" != "0" ] && [ -f "${lock_file}" ]; then
|
|
log_run "Lock file present. Let's wait ${lock_wait} and check again."
|
|
sleep "${lock_wait}"
|
|
fi
|
|
|
|
if [ -f "${lock_file}" ]; then
|
|
log_abort_and_quit "Lock file still present."
|
|
else
|
|
log_run "Lock file absent. Let's put one."
|
|
touch "${lock_file}"
|
|
fi
|
|
}
|
|
|
|
# If a script has been run in the ast 30 minutes, running it again won't fix the issue.
|
|
# We use a /run/ausosysadmin/${PROGNAME}_lastrun file to keep track of this.
|
|
#
|
|
# This can by bypassed in interactive mode.
|
|
# This is bypassed in debug mode.
|
|
# It's OK to lose this data after a reboot.
|
|
ensure_not_too_soon_or_exit() {
|
|
if is_debug; then
|
|
log_run "Debug mode enabled: continue without checking when was the last run."
|
|
return 0;
|
|
fi
|
|
|
|
lastrun_file="${RUN_DIR}/${PROGNAME}_lastrun"
|
|
if [ -f "${lastrun_file}" ]; then
|
|
lastrun_age="$(($(date +%s)-$(stat -c "%Y" "${lastrun_file}")))"
|
|
log_run "Last run was ${lastrun_age} seconds ago."
|
|
if [ "${lastrun_age}" -lt 1800 ]; then
|
|
if is_interactive; then
|
|
echo "${PROGNAME} was run ${lastrun_age} seconds ago."
|
|
answer=""
|
|
while :; do
|
|
printf "> Continue? [Y,n,?] "
|
|
read -r answer
|
|
case ${answer} in
|
|
[Yy]|"" )
|
|
log_run "Last run check bypassed manually in interactive mode."
|
|
break
|
|
;;
|
|
[Nn] )
|
|
log_run "Last run check confirmed manually in interactive mode."
|
|
log_abort_and_quit 'Last run too recent.'
|
|
;;
|
|
* )
|
|
printf "y - yes, continue\n"
|
|
printf "n - no, exit\n"
|
|
printf "? - print this help\n"
|
|
;;
|
|
esac
|
|
done
|
|
else
|
|
log_abort_and_quit "Last run too recent."
|
|
fi
|
|
fi
|
|
fi
|
|
touch "${lastrun_file}"
|
|
}
|
|
|
|
# Populate DEBIAN_VERSION and DEBIAN_RELEASE variables
|
|
# based on gathered information about the operating system
|
|
detect_os() {
|
|
DEBIAN_RELEASE="unknown"
|
|
DEBIAN_VERSION="unknown"
|
|
LSB_RELEASE_BIN="$(command -v lsb_release)"
|
|
|
|
if [ -e /etc/debian_version ]; then
|
|
DEBIAN_VERSION="$(cut -d "." -f 1 < /etc/debian_version)"
|
|
if [ -x "${LSB_RELEASE_BIN}" ]; then
|
|
DEBIAN_RELEASE="$("${LSB_RELEASE_BIN}" --codename --short)"
|
|
else
|
|
case "${DEBIAN_VERSION}" in
|
|
7) DEBIAN_RELEASE="wheezy" ;;
|
|
8) DEBIAN_RELEASE="jessie" ;;
|
|
9) DEBIAN_RELEASE="stretch" ;;
|
|
10) DEBIAN_RELEASE="buster" ;;
|
|
11) DEBIAN_RELEASE="bullseye" ;;
|
|
12) DEBIAN_RELEASE="bookworm" ;;
|
|
13) DEBIAN_RELEASE="trixie" ;;
|
|
esac
|
|
fi
|
|
# log_run "Detected OS: Debian version=${DEBIAN_VERSION} release=${DEBIAN_RELEASE}"
|
|
# else
|
|
# log_run "Detected OS: unknown (missing /etc/debian_version)"
|
|
fi
|
|
}
|
|
|
|
is_debian_wheezy() {
|
|
test "${DEBIAN_RELEASE}" = "wheezy"
|
|
}
|
|
is_debian_jessie() {
|
|
test "${DEBIAN_RELEASE}" = "jessie"
|
|
}
|
|
is_debian_stretch() {
|
|
test "${DEBIAN_RELEASE}" = "stretch"
|
|
}
|
|
is_debian_buster() {
|
|
test "${DEBIAN_RELEASE}" = "buster"
|
|
}
|
|
is_debian_bullseye() {
|
|
test "${DEBIAN_RELEASE}" = "bullseye"
|
|
}
|
|
is_debian_bookworm() {
|
|
test "${DEBIAN_RELEASE}" = "bookworm"
|
|
}
|
|
is_debian_trixie() {
|
|
test "${DEBIAN_RELEASE}" = "trixie"
|
|
}
|
|
is_debian_version() {
|
|
local version=$1
|
|
local relation=${2:-"eq"}
|
|
|
|
if [ -z "${DEBIAN_VERSION:-""}" ]; then
|
|
detect_os
|
|
fi
|
|
|
|
dpkg --compare-versions "${DEBIAN_VERSION}" "${relation}" "${version}"
|
|
}
|
|
|
|
# List systemd services (only names), even if stopped
|
|
systemd_list_services() {
|
|
pattern=$1
|
|
|
|
systemctl list-units --all --no-legend --type=service "${pattern}" | grep --only-matching --extended-regexp '\S+\.service'
|
|
}
|
|
|
|
is_systemd_enabled() {
|
|
systemctl --quiet is-enabled "$1" 2> /dev/null
|
|
}
|
|
|
|
is_systemd_active() {
|
|
systemctl --quiet is-active "$1" 2> /dev/null
|
|
}
|
|
|
|
is_sysvinit_enabled() {
|
|
find /etc/rc2.d/ -name "$1" > /dev/null
|
|
}
|
|
|
|
get_fqdn() {
|
|
# shellcheck disable=SC2155
|
|
local system=$(uname -s)
|
|
|
|
if [ "${system}" = "Linux" ]; then
|
|
hostname --fqdn
|
|
elif [ "${system}" = "OpenBSD" ]; then
|
|
hostname
|
|
else
|
|
log_abort_and_quit "System '${system}' not recognized."
|
|
fi
|
|
}
|
|
|
|
get_complete_hostname() {
|
|
REAL_HOSTNAME="$(get_fqdn)"
|
|
if [ "${HOSTNAME}" = "${REAL_HOSTNAME}" ]; then
|
|
echo "${HOSTNAME}"
|
|
else
|
|
echo "${HOSTNAME} (${REAL_HOSTNAME})"
|
|
fi
|
|
}
|
|
# Fetch values from evomaintenance configuration
|
|
get_evomaintenance_mail() {
|
|
grep "EVOMAINTMAIL=" /etc/evomaintenance.cf | cut -d '=' -f2
|
|
}
|
|
get_evomaintenance_emergency_mail() {
|
|
grep "URGENCYFROM=" /etc/evomaintenance.cf | cut -d '=' -f2
|
|
}
|
|
get_evomaintenance_emergency_tel() {
|
|
grep "URGENCYTEL=" /etc/evomaintenance.cf | cut -d '=' -f2
|
|
}
|
|
|
|
# Log a message to the log file in the log directory
|
|
log_run() {
|
|
local msg="${1:-$(cat /dev/stdin)}"
|
|
# shellcheck disable=SC2155
|
|
local date=$(/bin/date +"${DATE_FORMAT}")
|
|
|
|
printf "[%s] %s[%s]: %s\\n" \
|
|
"${date}" "${PROGNAME}" "${PID}" "${msg}" \
|
|
>> "${RUN_LOG_FILE}"
|
|
}
|
|
# Log a message in the system log file (syslog or journald)
|
|
log_global() {
|
|
local msg="${1:-$(cat /dev/stdin)}"
|
|
|
|
echo "${msg}" \
|
|
| /usr/bin/logger -p local0.notice -t autosysadmin
|
|
}
|
|
# Log a message in both places
|
|
log_all() {
|
|
local msg="${1:-$(cat /dev/stdin)}"
|
|
|
|
log_global "${msg}"
|
|
log_run "${msg}"
|
|
}
|
|
# Log a notable action in regular places
|
|
# and append it to the dedicated list
|
|
log_action() {
|
|
log_all "$*"
|
|
append_action "$*"
|
|
}
|
|
# Append a line in the actions.log file in the log directory
|
|
append_action() {
|
|
echo "$*" >> "${ACTIONS_FILE}"
|
|
}
|
|
# Print the content of the actions.log file
|
|
# or a fallback content (1st parameter) if empty
|
|
# shellcheck disable=SC2120
|
|
print_actions() {
|
|
local fallback=${1:-""}
|
|
if [ -s "${ACTIONS_FILE}" ]; then
|
|
cat "${ACTIONS_FILE}"
|
|
elif [ -n "${fallback}" ]; then
|
|
echo "${fallback}"
|
|
fi
|
|
}
|
|
|
|
# Log a an abort reason in regular places
|
|
# and append it to the dedicated list
|
|
log_abort() {
|
|
log_all "$*"
|
|
append_abort_reason "$*"
|
|
}
|
|
# Append a line in the abort.log file in the log directory
|
|
append_abort_reason() {
|
|
echo "$*" >> "${ABORT_FILE}"
|
|
}
|
|
# Print the content of the abort.log file
|
|
# or a fallback content (1st parameter) if empty
|
|
# shellcheck disable=SC2120
|
|
print_abort_reasons() {
|
|
local fallback=${1:-""}
|
|
if [ -s "${ABORT_FILE}" ]; then
|
|
cat "${ABORT_FILE}"
|
|
elif [ -n "${fallback}" ]; then
|
|
echo "${fallback}"
|
|
fi
|
|
}
|
|
# Print the content of the main log from the log directory
|
|
print_main_log() {
|
|
cat "${RUN_LOG_FILE}"
|
|
}
|
|
# Log an abort reason and quit the script
|
|
log_abort_and_quit() {
|
|
log_abort "$*"
|
|
quit
|
|
}
|
|
|
|
# Store the content from standard inpu
|
|
# into a file in the log directory named after the 1st parameter
|
|
save_in_log_dir() {
|
|
local file_name=$1
|
|
local file_path="${RUN_LOG_DIR}/${file_name}"
|
|
|
|
cat /dev/stdin > "${file_path}"
|
|
|
|
log_run "Saved \`${file_name}' file."
|
|
}
|
|
# Return the full path of the file in log directory
|
|
# based on the name in the 1st parameter
|
|
file_path_in_log_dir() {
|
|
echo "${RUN_LOG_DIR}/${1}"
|
|
}
|
|
|
|
format_mail_success() {
|
|
cat <<EOTEMPLATE
|
|
From: AutoSysadmin Evolix <${EMAIL_FROM}>
|
|
Content-Type: text/plain; charset=UTF-8
|
|
MIME-Version: 1.0
|
|
Content-Transfer-Encoding: 8bit
|
|
X-Script: ${PROGNAME}
|
|
X-RunId: ${RUN_ID}
|
|
To: ${EMAIL_CLIENT:-alert5@evolix.fr}
|
|
Cc: ${EMAIL_INTERNAL}
|
|
Subject: [autosysadmin] Intervention automatisée sur ${HOSTNAME_TEXT}
|
|
|
|
Bonjour,
|
|
|
|
Une intervention automatisée vient de se terminer.
|
|
|
|
Nom du serveur : ${HOSTNAME_TEXT}
|
|
Heure d'intervention : $(LC_ALL=fr_FR.utf8 date)
|
|
Script déclenché : ${PROGNAME}
|
|
|
|
### Actions réalisées
|
|
|
|
$(print_actions "Aucune")
|
|
|
|
### Réagir à cette intervention
|
|
|
|
Vous pouvez répondre à ce message (${EMAIL_FROM}).
|
|
|
|
En cas d'urgence, utilisez l'adresse ${EMERGENCY_MAIL}
|
|
ou notre ligne d'astreinte (${EMERGENCY_TEL})
|
|
|
|
--
|
|
Votre AutoSysadmin
|
|
EOTEMPLATE
|
|
}
|
|
|
|
format_mail_abort() {
|
|
cat <<EOTEMPLATE
|
|
From: AutoSysadmin Evolix <${EMAIL_FROM}>
|
|
Content-Type: text/plain; charset=UTF-8
|
|
MIME-Version: 1.0
|
|
Content-Transfer-Encoding: 8bit
|
|
X-Script: ${PROGNAME}
|
|
X-RunId: ${RUN_ID}
|
|
To: ${EMAIL_CLIENT:-alert5@evolix.fr}
|
|
Cc: ${EMAIL_INTERNAL}
|
|
Subject: [autosysadmin] Intervention automatisée interrompue sur ${HOSTNAME_TEXT}
|
|
|
|
Bonjour,
|
|
|
|
Une intervention automatisée a été déclenchée mais s'est interrompue.
|
|
|
|
Nom du serveur : ${HOSTNAME_TEXT}
|
|
Heure d'intervention : $(LC_ALL=fr_FR.utf8 date)
|
|
Script déclenché : ${PROGNAME}
|
|
|
|
### Actions réalisées
|
|
|
|
$(print_actions "Aucune")
|
|
|
|
### Raison(s) de l'interruption
|
|
|
|
$(print_abort_reasons "Inconnue")
|
|
|
|
### Réagir à cette intervention
|
|
|
|
Vous pouvez répondre à ce message (${EMAIL_FROM}).
|
|
|
|
En cas d'urgence, utilisez l'adresse ${EMERGENCY_MAIL}
|
|
ou notre ligne d'astreinte (${EMERGENCY_TEL})
|
|
|
|
--
|
|
Votre AutoSysadmin
|
|
EOTEMPLATE
|
|
}
|
|
|
|
# shellcheck disable=SC2028
|
|
print_report_information() {
|
|
echo "**Uptime**"
|
|
echo ""
|
|
uptime
|
|
|
|
echo ""
|
|
echo "**Utilisateurs récents**"
|
|
echo ""
|
|
who_file=$(file_path_in_log_dir "who-users")
|
|
if [ -s "${who_file}" ]; then
|
|
cat "${who_file}"
|
|
else
|
|
who --users
|
|
fi
|
|
|
|
echo ""
|
|
echo "**Espace disque**"
|
|
echo ""
|
|
df_file=$(file_path_in_log_dir "server-state/df.txt")
|
|
if [ -s "${df_file}" ]; then
|
|
cat "${df_file}"
|
|
else
|
|
df -h
|
|
fi
|
|
|
|
echo ""
|
|
echo "**Dmesg**"
|
|
echo ""
|
|
dmesg_file=$(file_path_in_log_dir "server-state/dmesg.txt")
|
|
if [ -s "${dmesg_file}" ]; then
|
|
tail -n 5 "${dmesg_file}"
|
|
else
|
|
dmesg | tail -n 5
|
|
fi
|
|
|
|
echo ""
|
|
echo "**systemd failed services**"
|
|
echo ""
|
|
failed_services_file=$(file_path_in_log_dir "server-state/systemctl-failed-services.txt")
|
|
if [ -s "${failed_services_file}" ]; then
|
|
cat "${failed_services_file}"
|
|
else
|
|
systemctl --no-legend --state=failed --type=service
|
|
fi
|
|
|
|
if command -v lxc-ls > /dev/null 2>&1; then
|
|
echo ""
|
|
echo "**LXC containers**"
|
|
echo ""
|
|
lxc_ls_file=$(file_path_in_log_dir "server-state/lxc-list.txt")
|
|
if [ -s "${lxc_ls_file}" ]; then
|
|
cat "${lxc_ls_file}"
|
|
else
|
|
lxc-ls --fancy
|
|
fi
|
|
fi
|
|
|
|
apache_errors_file=$(file_path_in_log_dir "apache-errors.log")
|
|
if [ -f "${apache_errors_file}" ]; then
|
|
echo ""
|
|
echo "**Apache errors**"
|
|
echo ""
|
|
cat "${apache_errors_file}"
|
|
fi
|
|
|
|
nginx_errors_file=$(file_path_in_log_dir "nginx-errors.log")
|
|
if [ -f "${nginx_errors_file}" ]; then
|
|
echo ""
|
|
echo "**Nginx errors**"
|
|
echo ""
|
|
cat "${nginx_errors_file}"
|
|
fi
|
|
}
|
|
|
|
format_mail_internal() {
|
|
cat <<EOTEMPLATE
|
|
From: AutoSysadmin Evolix <${EMAIL_FROM}>
|
|
Content-Type: text/plain; charset=UTF-8
|
|
MIME-Version: 1.0
|
|
Content-Transfer-Encoding: 8bit
|
|
X-Script: ${PROGNAME}
|
|
X-RunId: ${RUN_ID}
|
|
To: ${EMAIL_INTERNAL}
|
|
Subject: [autosysadmin] Rapport interne d'intervention sur ${HOSTNAME_TEXT}
|
|
|
|
Bonjour,
|
|
|
|
Une intervention automatique vient de se terminer.
|
|
|
|
Nom du serveur : ${HOSTNAME_TEXT}
|
|
Heure d'intervention : $(LC_ALL=fr_FR.utf8 date)
|
|
Script déclenché : ${PROGNAME}
|
|
|
|
### Actions réalisées
|
|
|
|
$(print_actions "Aucune")
|
|
|
|
### Raison(s) de l'interruption
|
|
|
|
$(print_abort_reasons "Aucune")
|
|
|
|
### Log autosysadmin
|
|
|
|
$(print_main_log)
|
|
|
|
### Informations additionnelles
|
|
|
|
$(print_report_information)
|
|
|
|
--
|
|
Votre AutoSysadmin
|
|
EOTEMPLATE
|
|
}
|
|
|
|
# Generic function to send emails at the end of the script.
|
|
# Takes a template as 1st parameter
|
|
hook_mail() {
|
|
if is_debug; then
|
|
log_run "Debug mode enabled: continue without sending mail."
|
|
return 0;
|
|
fi
|
|
|
|
HOSTNAME="${HOSTNAME:-"$(get_fqdn)"}"
|
|
HOSTNAME_TEXT="$(get_complete_hostname)"
|
|
EMAIL_CLIENT="$(get_evomaintenance_mail)"
|
|
EMERGENCY_MAIL="$(get_evomaintenance_emergency_mail)"
|
|
EMERGENCY_TEL="$(get_evomaintenance_emergency_tel)"
|
|
|
|
MAIL_CONTENT="$(format_mail_"$1")"
|
|
|
|
SENDMAIL_BIN="$(command -v sendmail)"
|
|
|
|
if [ -z "${SENDMAIL_BIN}" ]; then
|
|
log_global "ERROR: No \`sendmail' command has been found, can't send mail."
|
|
fi
|
|
if [ -x "${SENDMAIL_BIN}" ]; then
|
|
echo "${MAIL_CONTENT}" | "${SENDMAIL_BIN}" -oi -t -f "equipe@evolix.fr"
|
|
log_global "Sent '$1' mail for RUN_ID: ${RUN_ID}"
|
|
fi
|
|
}
|
|
|
|
is_holiday() {
|
|
# gcal mark today as a holiday by surrounding with < and > the day
|
|
# of the month of that holiday line. For example if today is 2022-05-01 we'll
|
|
# get among other lines:
|
|
# Fête du Travail (FR) + Di, < 1>Mai 2022
|
|
# Jour de la Victoire (FR) + Di, : 8:Mai 2022 = +7 jours
|
|
LANGUAGE=fr_FR.UTF-8 TZ=Europe/Paris gcal --cc-holidays=fr --holiday-list=short | grep -E '<[0-9 ]{2}>' --quiet
|
|
}
|
|
|
|
is_weekend() {
|
|
day_of_week=$(date +%u)
|
|
if [ "${day_of_week}" != 6 ] && [ "${day_of_week}" != 7 ]; then
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
is_workday() {
|
|
if is_holiday || is_weekend; then
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
is_worktime() {
|
|
if ! is_workday; then
|
|
return 1
|
|
fi
|
|
|
|
hour=$(date +%H)
|
|
if [ "${hour}" -lt 9 ] || { [ "${hour}" -ge 12 ] && [ "${hour}" -lt 14 ] ; } || [ "${hour}" -ge 18 ]; then
|
|
return 1
|
|
fi
|
|
}
|