rewrite (again) mysql functions

This commit is contained in:
Jérémy Lecour 2024-01-09 08:45:24 +01:00 committed by Jérémy Lecour
parent 01fb231437
commit 257679364d
Signed by: jlecour
SSH key fingerprint: SHA256:h+5LgHRKwN9lS0SsdVR5yZPeFlJE4Mt+8UtL4CcP8dY
2 changed files with 331 additions and 68 deletions

View file

@ -1,13 +1,6 @@
#!/bin/bash
#!/usr/bin/env bash
# shellcheck disable=SC2034,SC2317,SC2155
mysql_list_databases() {
port=${1:-"3306"}
mysql --defaults-extra-file=/etc/mysql/debian.cnf --port="${port}" --execute="show databases" --silent --skip-column-names \
| grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)"
}
#######################################################################
# Dump complete summary of an instance (using pt-mysql-summary)
#
@ -210,15 +203,16 @@ dump_mysql_summary() {
option_dump_label="${option_defaults_group_suffix}"
elif [ -n "${option_port}" ]; then
option_dump_label="${option_port}"
elif [ -n "${option_socket}" ]; then
option_dump_label=$(path_to_str "${option_socket}")
else
option_dump_label="default"
fi
fi
local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}"
local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}-summary"
local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}")
# DO NOT REMOVE EXISTING DIRECTORIES
# rm -rf "${dump_dir}" "${errors_dir}"
rm -rf "${dump_dir}" "${errors_dir}"
mkdir -p "${dump_dir}" "${errors_dir}"
# No need to change recursively, the top directory is enough
chmod 700 "${dump_dir}" "${errors_dir}"
@ -434,15 +428,16 @@ dump_mysql_grants() {
if [ -z "${option_dump_label}" ]; then
if [ -n "${option_port}" ]; then
option_dump_label="${option_port}"
elif [ -n "${option_socket}" ]; then
option_dump_label=$(path_to_str "${option_socket}")
else
option_dump_label="default"
fi
fi
local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}"
local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}-grants"
local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}")
# DO NOT REMOVE EXISTING DIRECTORIES
# rm -rf "${dump_dir}" "${errors_dir}"
rm -rf "${dump_dir}" "${errors_dir}"
mkdir -p "${dump_dir}" "${errors_dir}"
# No need to change recursively, the top directory is enough
chmod 700 "${dump_dir}" "${errors_dir}"
@ -491,6 +486,7 @@ dump_mysql_grants() {
#######################################################################
# Dump a single compressed file of all databases of an instance
# and a file containing only the schema.
#
# Arguments:
# --masterdata (default: <absent>)
@ -503,6 +499,8 @@ dump_mysql_grants() {
# --defaults-group-suffix=[String] (default: <blank>)
# --dump-label=[String] (default: "default")
# used as suffix of the dump dir to differenciate multiple instances
# --compress=<gzip|bzip2|xz|none> (default: "gzip")
# Other options after -- are passed as-is to mysqldump
#######################################################################
dump_mysql_global() {
local option_masterdata=""
@ -514,6 +512,8 @@ dump_mysql_global() {
local option_user=""
local option_password=""
local option_dump_label=""
local option_compress=""
local option_others=""
# Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a
while :; do
@ -673,9 +673,29 @@ dump_mysql_global() {
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument."
exit 1
;;
--compress)
# compress options, with value separated by space
if [ -n "$2" ]; then
option_compress="${2}"
shift
else
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument."
exit 1
fi
;;
--compress=?*)
# compress options, with value separated by =
option_compress="${1#*=}"
;;
--compress=)
# compress options, without value
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument."
exit 1
;;
--)
# End of all options.
shift
option_others=${*}
break
;;
-?*|[[:alnum:]]*)
@ -691,27 +711,37 @@ dump_mysql_global() {
shift
done
case "${option_compress}" in
none)
compress_cmd="cat"
dump_ext=""
;;
bzip2|bz|bz2)
compress_cmd="bzip2 --best"
dump_ext=".bz"
;;
xz)
compress_cmd="xz --best"
dump_ext=".xz"
;;
gz|gzip|*)
compress_cmd="gzip --best"
dump_ext=".gz"
;;
esac
if [ -z "${option_dump_label}" ]; then
if [ -n "${option_defaults_group_suffix}" ]; then
option_dump_label="${option_defaults_group_suffix}"
elif [ -n "${option_port}" ]; then
option_dump_label="${option_port}"
elif [ -n "${option_socket}" ]; then
option_dump_label=$(path_to_str "${option_socket}")
else
option_dump_label="default"
fi
fi
local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}"
local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}")
rm -rf "${dump_dir}" "${errors_dir}"
mkdir -p "${dump_dir}" "${errors_dir}"
# No need to change recursively, the top directory is enough
chmod 700 "${dump_dir}" "${errors_dir}"
local error_file="${errors_dir}/mysqldump.err"
local dump_file="${dump_dir}/mysqldump.sql.gz"
log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}"
## Connection options
declare -a connect_options
connect_options=()
@ -740,6 +770,18 @@ dump_mysql_global() {
fi
## Global all databases in one file
local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}"
local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}")
rm -rf "${dump_dir}" "${errors_dir}"
mkdir -p "${dump_dir}" "${errors_dir}"
# No need to change recursively, the top directory is enough
chmod 700 "${dump_dir}" "${errors_dir}"
local error_file="${errors_dir}/mysqldump.err"
local dump_file="${dump_dir}/mysqldump.sql${dump_ext}"
log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}"
declare -a dump_options
dump_options=()
dump_options+=(--opt)
@ -750,20 +792,29 @@ dump_mysql_global() {
if [ -n "${option_masterdata}" ]; then
dump_options+=("${option_masterdata}")
fi
if [ -n "${option_others}" ]; then
# word splitting is deliberate here
# shellcheck disable=SC2206
dump_options+=(${option_others})
fi
mysqldump "${connect_options[@]}" "${dump_options[@]}" 2> "${error_file}" | gzip --best > "${dump_file}"
dump_cmd="mysqldump ${connect_options[*]} ${dump_options[*]}"
log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd} | ${compress_cmd} > ${dump_file}"
${dump_cmd} 2> "${error_file}" | ${compress_cmd} > "${dump_file}"
local last_rc=$?
# shellcheck disable=SC2086
if [ ${last_rc} -ne 0 ]; then
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}"
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump returned an error ${last_rc}" "${error_file}"
GLOBAL_RC=${E_DUMPFAILED}
else
rm -f "${error_file}"
fi
log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}"
## Schema only (no data) for each databases
local error_file="${errors_dir}/mysqldump.schema.err"
local dump_file="${dump_dir}/mysqldump.schema.sql"
log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}"
@ -773,13 +824,20 @@ dump_mysql_global() {
dump_options+=(--force)
dump_options+=(--no-data)
dump_options+=(--all-databases)
if [ -n "${option_others}" ]; then
# word splitting is deliberate here
# shellcheck disable=SC2206
dump_options+=(${option_others})
fi
mysqldump "${connect_options[@]}" "${dump_options[@]}" 2> "${error_file}" > "${dump_file}"
dump_cmd="mysqldump ${connect_options[*]} ${dump_options[*]}"
log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd} > ${dump_file}"
${dump_cmd} 2> "${error_file}" > "${dump_file}"
local last_rc=$?
# shellcheck disable=SC2086
if [ ${last_rc} -ne 0 ]; then
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}"
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump returned an error ${last_rc}" "${error_file}"
GLOBAL_RC=${E_DUMPFAILED}
else
rm -f "${error_file}"
@ -788,7 +846,8 @@ dump_mysql_global() {
}
#######################################################################
# Dump a compressed file per database of an instance
# Dump a file of each databases of an instance
# and a file containing only the schema.
#
# Arguments:
# --port=[Integer] (default: <blank>)
@ -800,15 +859,81 @@ dump_mysql_global() {
# --defaults-group-suffix=[String] (default: <blank>)
# --dump-label=[String] (default: "default")
# used as suffix of the dump dir to differenciate multiple instances
# --compress=<gzip|bzip2|xz|none> (default: "gzip")
# Other options after -- are passed as-is to mysqldump
#######################################################################
dump_mysql_per_base() {
local option_port=""
local option_socket=""
local option_defaults_file=""
local option_defaults_extra_file=""
local option_defaults_group_suffix=""
local option_user=""
local option_password=""
local option_dump_label=""
local option_compress=""
local option_others=""
# Parse options, based on https://gist.github.com/deshion/10d3cb5f88a21671e17a
while :; do
case ${1:-''} in
--defaults-file)
# defaults-file options, with value separated by space
if [ -n "$2" ]; then
option_defaults_file="${2}"
shift
else
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument."
exit 1
fi
;;
--defaults-file=?*)
# defaults-file options, with value separated by =
option_defaults_file="${1#*=}"
;;
--defaults-file=)
# defaults-file options, without value
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument."
exit 1
;;
--defaults-extra-file)
# defaults-file options, with value separated by space
if [ -n "$2" ]; then
option_defaults_extra_file="${2}"
shift
else
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-file' requires a non-empty option argument."
exit 1
fi
;;
--defaults-extra-file=?*)
# defaults-extra-file options, with value separated by =
option_defaults_extra_file="${1#*=}"
;;
--defaults-extra-file=)
# defaults-extra-file options, without value
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-extra-file' requires a non-empty option argument."
exit 1
;;
--defaults-group-suffix)
# defaults-group-suffix options, with value separated by space
if [ -n "$2" ]; then
option_defaults_group_suffix="${2}"
shift
else
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-group-suffix' requires a non-empty option argument."
exit 1
fi
;;
--defaults-group-suffix=?*)
# defaults-group-suffix options, with value separated by =
option_defaults_group_suffix="${1#*=}"
;;
--defaults-group-suffix=)
# defaults-group-suffix options, without value
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--defaults-group-suffix' requires a non-empty option argument."
exit 1
;;
--port)
# port options, with value separated by space
if [ -n "$2" ]; then
@ -847,9 +972,86 @@ dump_mysql_per_base() {
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--socket' requires a non-empty option argument."
exit 1
;;
--user)
# user options, with value separated by space
if [ -n "$2" ]; then
option_user="${2}"
shift
else
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument."
exit 1
fi
;;
--user=?*)
# user options, with value separated by =
option_user="${1#*=}"
;;
--user=)
# user options, without value
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--user' requires a non-empty option argument."
exit 1
;;
--password)
# password options, with value separated by space
if [ -n "$2" ]; then
option_password="${2}"
shift
else
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument."
exit 1
fi
;;
--password=?*)
# password options, with value separated by =
option_password="${1#*=}"
;;
--password=)
# password options, without value
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--password' requires a non-empty option argument."
exit 1
;;
--dump-label)
# dump-label options, with value separated by space
if [ -n "$2" ]; then
option_dump_label="${2}"
shift
else
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument."
exit 1
fi
;;
--dump-label=?*)
# dump-label options, with value separated by =
option_dump_label="${1#*=}"
;;
--dump-label=)
# dump-label options, without value
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--dump-label' requires a non-empty option argument."
exit 1
;;
--compress)
# compress options, with value separated by space
if [ -n "$2" ]; then
option_compress="${2}"
shift
else
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument."
exit 1
fi
;;
--compress=?*)
# compress options, with value separated by =
option_compress="${1#*=}"
;;
--compress=)
# compress options, without value
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: '--compress' requires a non-empty option argument."
exit 1
;;
--)
# End of all options.
shift
option_others=${*}
break
;;
-?*|[[:alnum:]]*)
@ -865,89 +1067,138 @@ dump_mysql_per_base() {
shift
done
option_dump_label="${option_dump_label:-default}"
case "${option_compress}" in
none)
compress_cmd="cat"
dump_ext=""
;;
bzip2|bz|bz2)
compress_cmd="bzip2 --best"
dump_ext=".bz"
;;
xz)
compress_cmd="xz --best"
dump_ext=".xz"
;;
gz|gzip|*)
compress_cmd="gzip --best"
dump_ext=".gz"
;;
esac
if [ -z "${option_dump_label}" ]; then
if [ -n "${option_defaults_group_suffix}" ]; then
option_dump_label="${option_defaults_group_suffix}"
elif [ -n "${option_port}" ]; then
option_dump_label="${option_port}"
elif [ -n "${option_socket}" ]; then
option_dump_label=$(path_to_str "${option_socket}")
else
option_dump_label="default"
fi
fi
## Connection options
declare -a connect_options
connect_options=()
if [ -n "${option_defaults_file}" ]; then
connect_options+=(--defaults-file="${option_defaults_file}")
fi
if [ -n "${option_defaults_extra_file}" ]; then
connect_options+=(--defaults-extra-file="${option_defaults_extra_file}")
fi
if [ -n "${option_defaults_group_suffix}" ]; then
connect_options+=(--defaults-group-suffix="${option_defaults_group_suffix}")
fi
if [ -n "${option_port}" ]; then
connect_options+=(--protocol=tcp)
connect_options+=(--port="${option_port}")
fi
if [ -n "${option_socket}" ]; then
connect_options+=(--protocol=socket)
connect_options+=(--socket="${option_socket}")
fi
if [ -n "${option_user}" ]; then
connect_options+=(--user="${option_user}")
fi
if [ -n "${option_password}" ]; then
connect_options+=(--password="${option_password}")
fi
local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}/databases"
local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}-per-base"
local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}")
rm -rf "${dump_dir}" "${errors_dir}"
mkdir -p "${dump_dir}" "${errors_dir}"
# No need to change recursively, the top directory is enough
chmod 700 "${dump_dir}" "${errors_dir}"
declare -a options
options=()
options+=(--defaults-extra-file=/etc/mysql/debian.cnf)
options+=(--force)
options+=(--events)
options+=(--hex-blob)
databases=$(mysql_list_databases "${connect_options[@]}")
databases=$(mysql "${connect_options[@]}" --execute="show databases" --silent --skip-column-names \
| grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)")
for database in ${databases}; do
local error_file="${errors_dir}/${database}.err"
local dump_file="${dump_dir}/${database}.sql.gz"
local dump_file="${dump_dir}/${database}.sql${dump_ext}"
log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}"
mysqldump "${connect_options[@]}" "${options[@]}" "${database}" 2> "${error_file}" | gzip --best > "${dump_file}"
declare -a dump_options
dump_options=()
dump_options+=(--opt)
dump_options+=(--force)
dump_options+=(--events)
dump_options+=(--hex-blob)
dump_options+=(--databases "${database}")
if [ -n "${option_others}" ]; then
# word splitting is deliberate here
# shellcheck disable=SC2206
dump_options+=(${option_others})
fi
dump_cmd="mysqldump ${connect_options[*]} ${dump_options[*]}"
log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd} | ${compress_cmd} > ${dump_file}"
${dump_cmd} 2> "${error_file}" | ${compress_cmd} > "${dump_file}"
local last_rc=$?
# shellcheck disable=SC2086
if [ ${last_rc} -ne 0 ]; then
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}"
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump returned an error ${last_rc}" "${error_file}"
GLOBAL_RC=${E_DUMPFAILED}
else
rm -f "${error_file}"
fi
log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}"
done
## Schema only (no data) for each databases
for database in ${databases}; do
## Schema only (no data) for each databases
local error_file="${errors_dir}/${database}.schema.err"
local dump_file="${dump_dir}/${database}.schema.sql"
log "LOCAL_TASKS - ${FUNCNAME[0]}: start ${dump_file}"
declare -a options
options=()
options+=(--defaults-extra-file=/etc/mysql/debian.cnf)
options+=(--force)
options+=(--no-data)
options+=(--databases "${database}")
declare -a dump_options
dump_options=()
dump_options+=(--force)
dump_options+=(--no-data)
dump_options+=(--databases "${database}")
if [ -n "${option_others}" ]; then
# word splitting is deliberate here
# shellcheck disable=SC2206
dump_options+=(${option_others})
fi
mysqldump "${connect_options[@]}" "${options[@]}" 2> "${error_file}" > "${dump_file}"
dump_cmd="mysqldump ${connect_options[*]} ${dump_options[*]}"
log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd} > ${dump_file}"
${dump_cmd} 2> "${error_file}" > "${dump_file}"
local last_rc=$?
# shellcheck disable=SC2086
if [ ${last_rc} -ne 0 ]; then
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump to ${dump_file} returned an error ${last_rc}" "${error_file}"
log_error "LOCAL_TASKS - ${FUNCNAME[0]}: mysqldump returned an error ${last_rc}" "${error_file}"
GLOBAL_RC=${E_DUMPFAILED}
else
rm -f "${error_file}"
fi
log "LOCAL_TASKS - ${FUNCNAME[0]}: stop ${dump_file}"
done
## Grants and summary
if [ -n "${option_port}" ]; then
dump_mysql_grants --port="${option_port}"
dump_mysql_summary --port="${option_port}"
elif [ -n "${option_socket}" ]; then
dump_mysql_grants --socket="${option_socket}"
dump_mysql_summary --socket="${option_socket}"
else
dump_mysql_grants
dump_mysql_summary
fi
}
#######################################################################
@ -1031,10 +1282,11 @@ dump_mysql_tabs() {
option_dump_label="${option_dump_label:-default}"
databases=$(mysql_list_databases "${option_port}")
databases=$(mysql "${connect_options[@]}" --execute="show databases" --silent --skip-column-names \
| grep --extended-regexp --invert-match "^(Database|information_schema|performance_schema|sys)")
for database in ${databases}; do
local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}/tabs/${database}"
local dump_dir="${LOCAL_BACKUP_DIR}/mysql-${option_dump_label}-tabs/${database}"
local errors_dir=$(errors_dir_from_dump_dir "${dump_dir}")
rm -rf "${dump_dir}" "${errors_dir}"
mkdir -p "${dump_dir}" "${errors_dir}"
@ -1067,6 +1319,10 @@ dump_mysql_tabs() {
options+=("${database}")
mysqldump "${options[@]}" 2> "${error_file}"
dump_cmd="mysqldump ${connect_options[*]} ${dump_options[*]}"
log "LOCAL_TASKS - ${FUNCNAME[0]}: ${dump_cmd} > ${dump_file}"
${dump_cmd} 2> "${error_file}" > "${dump_file}"
local last_rc=$?
# shellcheck disable=SC2086

View file

@ -135,3 +135,10 @@ pick_server() {
send_mail() {
tail -20 "${LOGFILE}" | mail -s "${MAIL_SUBJECT}" "${MAIL}"
}
path_to_str() {
local path=$1
local str="${path}"
echo "${path}" | sed -e 's|^/||; s|/$||; s|/|:|g'
}