Managing SSH chroots to backup a lot of machines
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

444 lines
18 KiB

  1. #!/bin/sh
  2. #
  3. # Script Evobackup client
  4. # See https://gitea.evolix.org/evolix/evobackup
  5. #
  6. # Author: Gregory Colpart <reg@evolix.fr>
  7. # Contributors:
  8. # Romain Dessort <rdessort@evolix.fr>
  9. # Benoît Série <bserie@evolix.fr>
  10. # Tristan Pilat <tpilat@evolix.fr>
  11. # Victor Laborie <vlaborie@evolix.fr>
  12. # Jérémy Lecour <jlecour@evolix.fr>
  13. #
  14. # Licence: AGPLv3
  15. #
  16. # /!\ DON'T FORGET TO SET "MAIL" and "SERVERS" VARIABLES
  17. # Fail on unassigned variables
  18. set -u
  19. ##### Configuration ###################################################
  20. # email adress for notifications
  21. MAIL=jdoe@example.com
  22. # list of hosts (hostname or IP) and SSH port for Rsync
  23. SERVERS="node0.backup.example.com:2XXX node1.backup.example.com:2XXX"
  24. # Should we fallback on servers when the first is unreachable ?
  25. SERVERS_FALLBACK=${SERVERS_FALLBACK:-1}
  26. # timeout (in seconds) for SSH connections
  27. SSH_CONNECT_TIMEOUT=${SSH_CONNECT_TIMEOUT:-90}
  28. ## We use /home/backup : feel free to use your own dir
  29. LOCAL_BACKUP_DIR="/home/backup"
  30. # You can set "linux" or "bsd" manually or let it choose automatically
  31. SYSTEM=$(uname | tr '[:upper:]' '[:lower:]')
  32. # Change these 2 variables if you have more than one backup cron
  33. PIDFILE="/var/run/evobackup.pid"
  34. LOGFILE="/var/log/evobackup.log"
  35. ## Enable/Disable tasks
  36. LOCAL_TASKS=${LOCAL_TASKS:-1}
  37. SYNC_TASKS=${SYNC_TASKS:-1}
  38. ##### SETUP AND FUNCTIONS #############################################
  39. BEGINNING=$(/bin/date +"%d-%m-%Y ; %H:%M")
  40. # shellcheck disable=SC2174
  41. mkdir -p -m 700 ${LOCAL_BACKUP_DIR}
  42. PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin
  43. ## lang = C for english outputs
  44. export LANGUAGE=C
  45. export LANG=C
  46. ## Force umask
  47. umask 077
  48. ## Initialize variable to store SSH connection errors
  49. SERVERS_SSH_ERRORS=""
  50. # Call test_server with "HOST:PORT" string
  51. # It will return with 0 if the server is reachable.
  52. # It will return with 1 and a message on stderr if not.
  53. test_server() {
  54. item=$1
  55. # split HOST and PORT from the input string
  56. host=$(echo "${item}" | cut -d':' -f1)
  57. port=$(echo "${item}" | cut -d':' -f2)
  58. # Test if the server is accepting connections
  59. ssh -q -o "ConnectTimeout ${SSH_CONNECT_TIMEOUT}" "${host}" -p "${port}" -t "exit"
  60. # shellcheck disable=SC2181
  61. if [ $? = 0 ]; then
  62. # SSH connection is OK
  63. return 0
  64. else
  65. # SSH connection failed
  66. new_error=$(printf "Failed to connect to \`%s' within %s seconds" "${item}" "${SSH_CONNECT_TIMEOUT}")
  67. SERVERS_SSH_ERRORS=$(printf "%s\\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d')
  68. return 1
  69. fi
  70. }
  71. # Call pick_server with an optional positive integer to get the nth server in the list.
  72. pick_server() {
  73. increment=${1:-0}
  74. list_length=$(echo "${SERVERS}" | wc -w)
  75. if [ "${increment}" -ge "${list_length}" ]; then
  76. # We've reached the end of the list
  77. new_error="No more server available"
  78. SERVERS_SSH_ERRORS=$(printf "%s\\n%s" "${SERVERS_SSH_ERRORS}" "${new_error}" | sed -e '/^$/d')
  79. # Log errors to stderr
  80. printf "%s\\n" "${SERVERS_SSH_ERRORS}" >&2
  81. # Log errors to logfile
  82. printf "%s\\n" "${SERVERS_SSH_ERRORS}" >> $LOGFILE
  83. return 1
  84. fi
  85. # Extract the day of month, without leading 0 (which would give an octal based number)
  86. today=$(date +%e)
  87. # A salt is useful to randomize the starting point in the list
  88. # but stay identical each time it's called for a server (based on hostname).
  89. salt=$(hostname | cksum | cut -d' ' -f1)
  90. # Pick an integer between 0 and the length of the SERVERS list
  91. # It changes each day
  92. item=$(( (today + salt + increment) % list_length ))
  93. # cut starts counting fields at 1, not 0.
  94. field=$(( item + 1 ))
  95. echo "${SERVERS}" | cut -d' ' -f${field}
  96. }
  97. ## Verify other evobackup process and kill if needed
  98. if [ -e "${PIDFILE}" ]; then
  99. pid=$(cat "${PIDFILE}")
  100. # Does process still exist ?
  101. if kill -0 "${pid}" 2> /dev/null; then
  102. # Killing the childs of evobackup.
  103. for ppid in $(pgrep -P "${pid}"); do
  104. kill -9 "${ppid}";
  105. done
  106. # Then kill the main PID.
  107. kill -9 "${pid}"
  108. printf "%s is still running (PID %s). Process has been killed" "$0" "${pid}\\n" >&2
  109. else
  110. rm -f ${PIDFILE}
  111. fi
  112. fi
  113. echo "$$" > ${PIDFILE}
  114. # shellcheck disable=SC2064
  115. trap "rm -f ${PIDFILE}" EXIT
  116. ##### LOCAL BACKUP ####################################################
  117. if [ "${LOCAL_TASKS}" = "1" ]; then
  118. # You can comment or uncomment sections below to customize the backup
  119. ## OpenLDAP : example with slapcat
  120. # slapcat -l ${LOCAL_BACKUP_DIR}/ldap.bak
  121. ## MySQL
  122. ## example with global and compressed mysqldump
  123. # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
  124. # --opt --all-databases --force --events --hex-blob | gzip --best > ${LOCAL_BACKUP_DIR}/mysql.bak.gz
  125. ## example with two dumps for each table (.sql/.txt) for all databases
  126. # for i in $(echo SHOW DATABASES | mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 \
  127. # | egrep -v "^(Database|information_schema|performance_schema|sys)" ); \
  128. # do mkdir -p -m 700 /home/mysqldump/$i ; chown -RL mysql /home/mysqldump ; \
  129. # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 -Q --opt --events --hex-blob --skip-comments \
  130. # --fields-enclosed-by='\"' --fields-terminated-by=',' -T /home/mysqldump/$i $i; done
  131. ## Dump all grants (requires 'percona-toolkit' package)
  132. # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
  133. # pt-show-grants --flush --no-header > ${LOCAL_BACKUP_DIR}/mysql/all_grants.sql
  134. ## example with SQL dump (schema only, no data) for each databases
  135. # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
  136. # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
  137. # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
  138. # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --no-data --databases $i > ${LOCAL_BACKUP_DIR}/mysql/${i}.schema.sql
  139. # done
  140. ## example with compressed SQL dump (with data) for each databases
  141. # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/
  142. # for i in $(mysql --defaults-extra-file=/etc/mysql/debian.cnf -P 3306 -e 'show databases' -s --skip-column-names \
  143. # | egrep -v "^(Database|information_schema|performance_schema|sys)"); do
  144. # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -P 3306 --events --hex-blob $i | gzip --best > ${LOCAL_BACKUP_DIR}/mysql/${i}.sql.gz
  145. # done
  146. ## example with *one* uncompressed SQL dump for *one* database (MYBASE)
  147. # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysql/MYBASE
  148. # chown -RL mysql ${LOCAL_BACKUP_DIR}/mysql/
  149. # mysqldump --defaults-extra-file=/etc/mysql/debian.cnf --force -Q \
  150. # --opt --events --hex-blob --skip-comments -T ${LOCAL_BACKUP_DIR}/mysql/MYBASE MYBASE
  151. ## example with mysqlhotcopy
  152. # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mysqlhotcopy/
  153. # mysqlhotcopy BASE ${LOCAL_BACKUP_DIR}/mysql/mysqlhotcopy/
  154. ## example for multiples MySQL instances
  155. # mysqladminpasswd=$(grep -m1 'password = .*' /root/.my.cnf|cut -d" " -f3)
  156. # grep -E "^port\s*=\s*\d*" /etc/mysql/my.cnf |while read instance; do
  157. # instance=$(echo "$instance"|awk '{ print $3 }')
  158. # if [ "$instance" != "3306" ]
  159. # then
  160. # mysqldump -P $instance --opt --all-databases --hex-blob -u mysqladmin -p$mysqladminpasswd > ${LOCAL_BACKUP_DIR}/mysql.$instance.bak
  161. # fi
  162. # done
  163. ## PostgreSQL
  164. ## example with pg_dumpall (warning: you need space in ~postgres)
  165. # su - postgres -c "pg_dumpall > ~/pg.dump.bak"
  166. # mv ~postgres/pg.dump.bak ${LOCAL_BACKUP_DIR}/
  167. ## another method with gzip directly piped
  168. # cd /var/lib/postgresql
  169. # sudo -u postgres pg_dumpall | gzip > ${LOCAL_BACKUP_DIR}/pg.dump.bak.gz
  170. # cd - > /dev/null
  171. ## example with all tables from MYBASE excepts TABLE1 and TABLE2
  172. # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -t 'TABLE1' -t 'TABLE2' MYBASE
  173. ## example with only TABLE1 and TABLE2 from MYBASE
  174. # pg_dump -p 5432 -h 127.0.0.1 -U USER --clean -F t --inserts -f ${LOCAL_BACKUP_DIR}/pg-backup.tar -T 'TABLE1' -T 'TABLE2' MYBASE
  175. ## MongoDB
  176. ## don't forget to create use with read-only access
  177. ## > use admin
  178. ## > db.createUser( { user: "mongobackup", pwd: "PASS", roles: [ "backup", ] } )
  179. # test -d ${LOCAL_BACKUP_DIR}/mongodump/ && rm -rf ${LOCAL_BACKUP_DIR}/mongodump/
  180. # mkdir -p -m 700 ${LOCAL_BACKUP_DIR}/mongodump/
  181. # mongodump --quiet -u mongobackup -pPASS -o ${LOCAL_BACKUP_DIR}/mongodump/
  182. # if [ $? -ne 0 ]; then
  183. # echo "Error with mongodump!"
  184. # fi
  185. ## Redis
  186. ## example with copy .rdb file
  187. ## for the default instance :
  188. # cp /var/lib/redis/dump.rdb ${LOCAL_BACKUP_DIR}/
  189. ## for multiple instances :
  190. # for instance in $(ls -d /var/lib/redis-*); do
  191. # name=$(basename $instance)
  192. # mkdir -p ${LOCAL_BACKUP_DIR}/${name}
  193. # cp -a ${instance}/dump.rdb ${LOCAL_BACKUP_DIR}/${name}
  194. # done
  195. ## ElasticSearch
  196. ## Take a snapshot as a backup.
  197. ## Warning: You need to have a path.repo configured.
  198. ## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes
  199. # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log
  200. # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log
  201. ## Clustered version here
  202. ## It basically the same thing except that you need to check that NFS is mounted
  203. # if ss | grep ':nfs' | grep -q 'ip\.add\.res\.s1' && ss | grep ':nfs' | grep -q 'ip\.add\.res\.s2'
  204. # then
  205. # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log
  206. # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log
  207. # else
  208. # echo 'Cannot make a snapshot of elasticsearch, at least one node is not mounting the repository.'
  209. # fi
  210. ## If you need to keep older snapshot, for example the last 10 daily snapshots, replace the XDELETE and XPUT lines by :
  211. # for snapshot in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep -Eo 'snapshot_[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -n -10); do
  212. # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/${snapshot}" | grep -v -Fx '{"acknowledged":true}'
  213. # done
  214. # date=$(date +%F)
  215. # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${date}?wait_for_completion=true" -o /tmp/es_snapshot_${date}.log
  216. ## RabbitMQ
  217. ## export config
  218. #rabbitmqadmin export ${LOCAL_BACKUP_DIR}/rabbitmq.config >> $LOGFILE
  219. ## MegaCli config
  220. #megacli -CfgSave -f ${LOCAL_BACKUP_DIR}/megacli_conf.dump -a0 >/dev/null
  221. ## Dump system and kernel versions
  222. uname -a > ${LOCAL_BACKUP_DIR}/uname
  223. ## Dump network routes with mtr and traceroute (warning: could be long with aggressive firewalls)
  224. for addr in 8.8.8.8 www.evolix.fr travaux.evolix.net; do
  225. mtr -r ${addr} > ${LOCAL_BACKUP_DIR}/mtr-${addr}
  226. traceroute -n ${addr} > ${LOCAL_BACKUP_DIR}/traceroute-${addr} 2>&1
  227. done
  228. ## Dump process with ps
  229. ps auwwx >${LOCAL_BACKUP_DIR}/ps.out
  230. if [ "${SYSTEM}" = "linux" ]; then
  231. ## Dump network connections with ss
  232. ss -taupen > ${LOCAL_BACKUP_DIR}/netstat.out
  233. ## List Debian packages
  234. dpkg -l > ${LOCAL_BACKUP_DIR}/packages
  235. dpkg --get-selections > ${LOCAL_BACKUP_DIR}/packages.getselections
  236. apt-cache dumpavail > ${LOCAL_BACKUP_DIR}/packages.available
  237. ## Dump MBR / table partitions
  238. disks=$(lsblk -l | grep disk | grep -v -E '(drbd|fd[0-9]+)' | awk '{print $1}')
  239. for disk in ${disks}; do
  240. dd if="/dev/${disk}" of="${LOCAL_BACKUP_DIR}/MBR-${disk}" bs=512 count=1 2>&1 | grep -Ev "(records in|records out|512 bytes)"
  241. fdisk -l "/dev/${disk}" > "${LOCAL_BACKUP_DIR}/partitions-${disk}" 2>&1
  242. done
  243. cat ${LOCAL_BACKUP_DIR}/partitions-* > ${LOCAL_BACKUP_DIR}/partitions
  244. ## Dump iptables
  245. if [ -x /sbin/iptables ]; then
  246. { /sbin/iptables -L -n -v; /sbin/iptables -t filter -L -n -v; } > ${LOCAL_BACKUP_DIR}/iptables.txt
  247. fi
  248. ## Dump findmnt(8) output
  249. FINDMNT_BIN=$(command -v findmnt)
  250. if [ -x "${FINDMNT_BIN}" ]; then
  251. ${FINDMNT_BIN} > ${LOCAL_BACKUP_DIR}/findmnt.txt
  252. fi
  253. else
  254. ## Dump network connections with netstat
  255. netstat -finet -atn > ${LOCAL_BACKUP_DIR}/netstat.out
  256. ## List OpenBSD packages
  257. pkg_info -m > ${LOCAL_BACKUP_DIR}/packages
  258. ## Dump MBR / table partitions
  259. disklabel sd0 > ${LOCAL_BACKUP_DIR}/partitions
  260. ## Dump pf infos
  261. pfctl -sa > ${LOCAL_BACKUP_DIR}/pfctl-sa.txt
  262. fi
  263. ## Dump rights
  264. #getfacl -R /var > ${LOCAL_BACKUP_DIR}/rights-var.txt
  265. #getfacl -R /etc > ${LOCAL_BACKUP_DIR}/rights-etc.txt
  266. #getfacl -R /usr > ${LOCAL_BACKUP_DIR}/rights-usr.txt
  267. #getfacl -R /home > ${LOCAL_BACKUP_DIR}/rights-home.txt
  268. fi
  269. ##### REMOTE BACKUP ###################################################
  270. n=0
  271. server=""
  272. if [ "${SERVERS_FALLBACK}" = "1" ]; then
  273. # We try to find a suitable server
  274. while :; do
  275. server=$(pick_server "${n}")
  276. test $? = 0 || exit 2
  277. if test_server "${server}"; then
  278. break
  279. else
  280. server=""
  281. n=$(( n + 1 ))
  282. fi
  283. done
  284. else
  285. # we force the server
  286. server=$(pick_server "${n}")
  287. fi
  288. SSH_SERVER=$(echo "${server}" | cut -d':' -f1)
  289. SSH_PORT=$(echo "${server}" | cut -d':' -f2)
  290. HOSTNAME=$(hostname)
  291. if [ "${SYSTEM}" = "linux" ]; then
  292. rep="/bin /boot /lib /opt /sbin /usr"
  293. else
  294. rep="/bsd /bin /sbin /usr"
  295. fi
  296. if [ "${SYNC_TASKS}" = "1" ]; then
  297. # /!\ DO NOT USE COMMENTS in the rsync command /!\
  298. # It breaks the command and destroys data, simply remove (or add) lines.
  299. # Remote shell command
  300. RSH_COMMAND="ssh -p ${SSH_PORT} -o 'ConnectTimeout ${SSH_CONNECT_TIMEOUT}'"
  301. # ignore check because we want it to split the different arguments to $rep
  302. # shellcheck disable=SC2086
  303. rsync -avzh --relative --stats --delete --delete-excluded --force --ignore-errors --partial \
  304. --exclude "dev" \
  305. --exclude "lost+found" \
  306. --exclude ".nfs.*" \
  307. --exclude "/usr/doc" \
  308. --exclude "/usr/obj" \
  309. --exclude "/usr/share/doc" \
  310. --exclude "/usr/src" \
  311. --exclude "/var/apt" \
  312. --exclude "/var/cache" \
  313. --exclude "/var/lib/amavis/amavisd.sock" \
  314. --exclude "/var/lib/amavis/tmp" \
  315. --exclude "/var/lib/clamav/*.tmp" \
  316. --exclude "/var/lib/elasticsearch" \
  317. --exclude "/var/lib/metche" \
  318. --exclude "/var/lib/munin/*tmp*" \
  319. --exclude "/var/lib/mysql" \
  320. --exclude "/var/lib/php5" \
  321. --exclude "/var/lib/php/sessions" \
  322. --exclude "/var/lib/postgres" \
  323. --exclude "/var/lib/postgresql" \
  324. --exclude "/var/lib/sympa" \
  325. --exclude "/var/lock" \
  326. --exclude "/var/log" \
  327. --exclude "/var/log/evobackup*" \
  328. --exclude "/var/run" \
  329. --exclude "/var/spool/postfix" \
  330. --exclude "/var/spool/squid" \
  331. --exclude "/var/state" \
  332. --exclude "lxc/*/rootfs/usr/doc" \
  333. --exclude "lxc/*/rootfs/usr/obj" \
  334. --exclude "lxc/*/rootfs/usr/share/doc" \
  335. --exclude "lxc/*/rootfs/usr/src" \
  336. --exclude "lxc/*/rootfs/var/apt" \
  337. --exclude "lxc/*/rootfs/var/cache" \
  338. --exclude "lxc/*/rootfs/var/lib/php5" \
  339. --exclude "lxc/*/rootfs/var/lock" \
  340. --exclude "lxc/*/rootfs/var/log" \
  341. --exclude "lxc/*/rootfs/var/run" \
  342. --exclude "lxc/*/rootfs/var/state" \
  343. --exclude "/home/mysqltmp" \
  344. ${rep} \
  345. /etc \
  346. /root \
  347. /var \
  348. /home \
  349. -e "${RSH_COMMAND}" \
  350. "root@${SSH_SERVER}:/var/backup/" \
  351. | tail -30 >> $LOGFILE
  352. fi
  353. ##### REPORTING #######################################################
  354. END=$(/bin/date +"%d-%m-%Y ; %H:%M")
  355. printf "EvoBackup - %s - START %s ON %s (LOCAL_TASKS=%s SYNC_TASKS=%s)\\n" \
  356. "${HOSTNAME}" "${BEGINNING}" "${SSH_SERVER}" "${LOCAL_TASKS}" "${SYNC_TASKS}" \
  357. >> $LOGFILE
  358. printf "EvoBackup - %s - STOP %s ON %s (LOCAL_TASKS=%s SYNC_TASKS=%s)\\n" \
  359. "${HOSTNAME}" "${END}" "${SSH_SERVER}" "${LOCAL_TASKS}" "${SYNC_TASKS}" \
  360. >> $LOGFILE
  361. tail -10 $LOGFILE | \
  362. mail -s "[info] EvoBackup - Client ${HOSTNAME}" \
  363. ${MAIL}