From 7b142965031205021ee298f67296b19b18bf28bb Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sat, 2 Oct 2021 12:50:01 +0200 Subject: [PATCH] etc-git: optimize maintenance tasks * manage commits with an optimized shell script instead of many slow Ansible tasks * centralize cron jobs in dedicated crontab --- CHANGELOG.md | 3 +- etc-git/files/etc-git-optimize | 11 ++ etc-git/files/etc-git-status | 11 ++ etc-git/files/evocommit | 227 ++++++++++++++++++++++++++++ etc-git/files/optimize-etc-git | 3 - etc-git/tasks/commit.yml | 34 ++++- etc-git/tasks/do_commit.yml | 77 ---------- etc-git/tasks/main.yml | 90 ++++++++--- etc-git/templates/etc-git-status.j2 | 4 - 9 files changed, 345 insertions(+), 115 deletions(-) create mode 100644 etc-git/files/etc-git-optimize create mode 100644 etc-git/files/etc-git-status create mode 100644 etc-git/files/evocommit delete mode 100644 etc-git/files/optimize-etc-git delete mode 100644 etc-git/tasks/do_commit.yml delete mode 100644 etc-git/templates/etc-git-status.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index db0c4b7a..78027f49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,8 @@ The **patch** part changes is incremented if multiple releases happen the same m * apache: new variable for mpm mode (+ updated default config accordingly) * certbot: add script for manual deploy hooks execution * docker-host: install additional dependencies -* etc-git: purge old .git/index.lock (default: True) +* etc-git: manage commits with an optimized shell script instead of many slow Ansible tasks +* etc-git: centralize cron jobs in dedicated crontab * evolinux-base: install molly-guard by default * generate-ldif: detect hardware raid card * generate-ldif: detect mdadm diff --git a/etc-git/files/etc-git-optimize b/etc-git/files/etc-git-optimize new file mode 100644 index 00000000..3d4932ee --- /dev/null +++ b/etc-git/files/etc-git-optimize @@ -0,0 +1,11 @@ +#!/bin/sh + +set -u + +repositories="/etc /etc/bind/ /usr/share/scripts" + +for repository in ${repositories}; do + if [ -d "${repository}/.git" ]; then + git --git-dir="${repository}/.git" gc --quiet + fi +done diff --git a/etc-git/files/etc-git-status b/etc-git/files/etc-git-status new file mode 100644 index 00000000..90603366 --- /dev/null +++ b/etc-git/files/etc-git-status @@ -0,0 +1,11 @@ +#!/bin/sh + +set -u + +repositories="/etc /etc/bind/ /usr/share/scripts" + +for repository in ${repositories}; do + if [ -d "${repository}/.git" ]; then + git --git-dir="${repository}/.git" --work-tree="${repository}" status --short + fi +done \ No newline at end of file diff --git a/etc-git/files/evocommit b/etc-git/files/evocommit new file mode 100644 index 00000000..0fd21076 --- /dev/null +++ b/etc-git/files/evocommit @@ -0,0 +1,227 @@ +#!/bin/sh + +set -u + +VERSION="21.10" + +show_version() { + cat <, + Jérémy Lecour + and others. + +evocommit comes with ABSOLUTELY NO WARRANTY. This is free software, +and you are welcome to redistribute it under certain conditions. +See the GNU General Public Licence for details. +END +} + +show_help() { + cat </dev/null; then + mountpoint=$(stat -c '%m' $1) + findmnt "${mountpoint}" --noheadings --output OPTIONS -O ro + else + grep /usr /proc/mounts | grep -E '\bro\b' + fi +} +remount_repository_readwrite() { + if [ "$(get_system)" = "OpenBSD" ]; then + partition=$(stat -f '%Sd' $1) + mount -u -w /dev/${partition} 2>/dev/null + else + mountpoint=$(stat -c '%m' $1) + mount -o remount,rw ${mountpoint} + syslog "Re-mount ${mountpoint} as read-write to commit in repository $1" + fi +} +remount_repository_readonly() { + if [ "$(get_system)" = "OpenBSD" ]; then + partition=$(stat -f '%Sd' $1) + mount -u -r /dev/${partition} 2>/dev/null + else + mountpoint=$(stat -c '%m' $1) + mount -o remount,ro ${mountpoint} 2>/dev/null + syslog "Re-mount ${mountpoint} as read-only after commit to repository $1" + fi +} +is_dry_run() { + test "${DRY_RUN}" = "1" +} +main() { + + lock="${GIT_DIR}/index.lock" + if [ -f "${lock}" ]; then + limit=$(date +"%s" -d "now - 1 hour") + updated_at=$(stat -c "%Y" "${lock}") + if [ "$updated_at" -lt "$limit" ]; then + rm -f "${lock}" + fi + fi + + git_status=$(${GIT_BIN} status --porcelain) + + if [ -n "${git_status}" ]; then + if is_dry_run; then + ${GIT_BIN} status + else + readonly_orig=0 + # remount mount point read-write if currently readonly + if is_repository_readonly "${REPOSITORY}"; then + readonly_orig=1; + remount_repository_readwrite "${REPOSITORY}"; + fi + author=$(logname) + email=$(git config --get user.email) + email=${email:-"${author}@evolix.net"} + # commit changes + ${GIT_BIN} add --all + ${GIT_BIN} commit --message "${MESSAGE}" --author "${author} <${email}>" --quiet + # remount mount point read-only if it was before + if [ ${readonly_orig} -eq 1 ]; then + remount_repository_readonly "${REPOSITORY}" + fi + fi + fi + + unset GIT_DIR + unset GIT_WORK_TREE +} +# Parse options +# based on https://gist.github.com/deshion/10d3cb5f88a21671e17a +while :; do + case ${1:-''} in + -h|-\?|--help) + show_help + exit 0 + ;; + -V|--version) + show_version + exit 0 + ;; + --message) + # message options, with value speparated by space + if [ -n "$2" ]; then + MESSAGE=$2 + shift + else + printf 'ERROR: "--message" requires a non-empty option argument.\n' >&2 + exit 1 + fi + ;; + --message=?*) + # message options, with value speparated by = + MESSAGE=${1#*=} + ;; + --message=) + # message options, without value + printf 'ERROR: "--message" requires a non-empty option argument.\n' >&2 + exit 1 + ;; + --repository) + # repository options, with value speparated by space + if [ -n "$2" ]; then + REPOSITORY=$2 + shift + else + printf 'ERROR: "--repository" requires a non-empty option argument.\n' >&2 + exit 1 + fi + ;; + --repository=?*) + # repository options, with value speparated by = + REPOSITORY=${1#*=} + ;; + --repository=) + # repository options, without value + printf 'ERROR: "--repository" requires a non-empty option argument.\n' >&2 + exit 1 + ;; + -n|--dry-run) + # disable actual commands + DRY_RUN=1 + ;; + -v|--verbose) + # print verbose information + VERBOSE=1 + ;; + --) + # End of all options. + shift + break + ;; + -?*|[[:alnum:]]*) + # ignore unknown options + printf 'WARN: Unknown option (ignored): %s\n' "$1" >&2 + ;; + *) + # Default case: If no more options then break out of the loop. + break + ;; + esac + + shift +done + +if [ -z "${MESSAGE}" ]; then + echo "Error: missing message parameter" >&2 + show_usage + exit 1 +fi +if [ -z "${REPOSITORY}" ]; then + echo "Error: missing repository parameter" >&2 + show_usage + exit 1 +fi +DRY_RUN=${DRY_RUN:-0} +VERBOSE=${VERBOSE:-0} + +GIT_BIN=$(command -v git) +readonly GIT_BIN + +LOGGER_BIN=$(command -v logger) +readonly LOGGER_BIN + +export GIT_DIR="${REPOSITORY}/.git" +export GIT_WORK_TREE="${REPOSITORY}" + +if [ -d "${GIT_DIR}" ]; then + main +else + echo "There is no Git repository in '${REPOSITORY}'" >&2 + exit 1 +fi \ No newline at end of file diff --git a/etc-git/files/optimize-etc-git b/etc-git/files/optimize-etc-git deleted file mode 100644 index a7b7510f..00000000 --- a/etc-git/files/optimize-etc-git +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -git --git-dir /etc/.git gc --quiet diff --git a/etc-git/tasks/commit.yml b/etc-git/tasks/commit.yml index 4bcf8e5c..f32c6ede 100644 --- a/etc-git/tasks/commit.yml +++ b/etc-git/tasks/commit.yml @@ -1,25 +1,47 @@ --- +- name: "evocommit script is installed" + copy: + src: evocommit + dest: /usr/local/bin/evocommit + mode: "0755" + force: yes + +# /etc - name: Is /etc a git repository stat: path: /etc/.git register: _etc_git -- include: do_commit.yml - vars: - git_folder: "/etc" +- name: "evocommit /etc" + command: "/usr/local/bin/evocommit --repository /etc --message \"{{ commit_message | mandatory }}\"" + ignore_errors: yes when: - _etc_git.stat.exists - _etc_git.stat.isdir +# /etc/bind +- name: Is /etc/bind a git repository + stat: + path: /etc/bind/.git + register: _etc_bind_git + +- name: "evocommit /etc/bind" + command: "/usr/local/bin/evocommit --repository /etc/bind --message \"{{ commit_message | mandatory }}\"" + ignore_errors: yes + when: + - _etc_bind_git.stat.exists + - _etc_bind_git.stat.isdir + +# /usr/share/scripts - name: Is /usr/share/scripts a git repository stat: path: /usr/share/scripts/.git register: _usr_share_scripts_git -- include: do_commit.yml - vars: - git_folder: "/usr/share/scripts" +- name: "evocommit /usr/share/scripts" + command: "/usr/local/bin/evocommit --repository /usr/share/scripts --message \"{{ commit_message | mandatory }}\"" + ignore_errors: yes when: - _usr_share_scripts_git.stat.exists - _usr_share_scripts_git.stat.isdir diff --git a/etc-git/tasks/do_commit.yml b/etc-git/tasks/do_commit.yml deleted file mode 100644 index bae36936..00000000 --- a/etc-git/tasks/do_commit.yml +++ /dev/null @@ -1,77 +0,0 @@ ---- - -- name: "Remount /usr if needed" - include_role: - name: remount-usr - when: git_folder is match('/usr/.*') - -- name: "stat {{ git_folder }}/.git/index.lock" - stat: - path: "{{ git_folder }}/.git/index.lock" - register: _index_lock - -- name: index file is removed if old enough - file: - path: "{{ git_folder }}/.git/index.lock" - state: absent - when: - - _index_lock.stat.exists - - _index_lock.stat.mtime | int <= ((ansible_date_time.epoch | int) - etc_git_purge_index_lock_age | default(86400)) - - etc_git_purge_index_lock_enabled - -- name: "is {{ git_folder }} clean?" - command: git status --porcelain - args: - chdir: "{{ git_folder }}" - changed_when: False - register: git_status - when: not ansible_check_mode - ignore_errors: yes - tags: - - etc-git - - commit - -- debug: - var: git_status - verbosity: 3 - tags: - - etc-git - - commit - -- name: fetch current Git user.email - git_config: - name: user.email - repo: "{{ git_folder }}" - register: git_config_user_email - ignore_errors: yes - tags: - - etc-git - - commit - -- name: "set commit author" - set_fact: - commit_author: '{% if ansible_env.SUDO_USER is not defined %}root{% else %}{{ ansible_env.SUDO_USER }}{% endif %}' - commit_email: '{% if git_config_user_email.config_value is not defined or not git_config_user_email.config_value %}root@localhost{% else %}{{ git_config_user_email.config_value }}{% endif %}' # noqa 204 - tags: - - etc-git - - commit - -- name: "{{ git_folder }} modifications are committed" - shell: "git add -A . && git commit -m \"{{ commit_message | mandatory }}\" --author \"{{ commit_author | mandatory }} <{{ commit_email | mandatory }}>\"" - args: - chdir: "{{ git_folder }}" - register: commit_end_run - when: - - not ansible_check_mode - - git_status.stdout | length > 0 - ignore_errors: yes - tags: - - etc-git - - commit - -- debug: - var: commit_end_run - verbosity: 4 - tags: - - etc-git - - commit diff --git a/etc-git/tasks/main.yml b/etc-git/tasks/main.yml index 37d1c692..90df9f60 100644 --- a/etc-git/tasks/main.yml +++ b/etc-git/tasks/main.yml @@ -32,6 +32,33 @@ - _usr_share_scripts.stat.isdir - ansible_distribution_major_version is version('10', '>=') +- name: "evocommit script is installed" + copy: + src: evocommit + dest: /usr/local/bin/evocommit + mode: "0755" + force: yes + tags: + - etc-git + +- name: "etc-git-optimize script is installed" + copy: + src: etc-git-optimize + dest: /usr/share/scripts/etc-git-optimize + mode: "0755" + force: yes + tags: + - etc-git + +- name: "etc-git-status script is installed" + copy: + src: etc-git-status + dest: /usr/share/scripts/etc-git-status + mode: "0755" + force: yes + tags: + - etc-git + - name: Check if cron is installed shell: "set -o pipefail && dpkg -l cron 2>/dev/null | grep -q -E '^(i|h)i'" args: @@ -41,29 +68,44 @@ check_mode: no register: is_cron_installed -- name: Optimize script is installed in monthly crontab - copy: - src: optimize-etc-git - dest: /etc/cron.monthly/optimize-etc-git - mode: "0750" - force: no +- block: + - name: Legacy cron jobs for /etc/.git status are absent + file: + dest: "{{ item }}" + state: absent + loop: + - /etc/cron.monthly/optimize-etc-git + - /etc/cron.d/etc-git-status + + - name: Cron job for monthly git optimization + cron: + name: "Monthly optimization" + cron_file: etc-git + special_time: "monthly" + user: root + job: "/usr/share/scripts/etc-git-optimize" + + - name: Cron job for hourly git status + cron: + name: "Hourly warning for unclean Git repository if nobody is connected" + cron_file: etc-git + special_time: "hourly" + user: root + job: "who > /dev/null || /usr/share/scripts/etc-git-status" + state: "{{ etc_git_monitor_status | bool | ternary('present','absent') }}" + + - name: Cron job for daily git status + cron: + name: "Daily warning for unclean Git repository" + cron_file: etc-git + user: root + job: "/usr/share/scripts/etc-git-status" + minute: "21" + hour: "21" + weekday: "*" + day: "*" + month: "*" + state: "{{ etc_git_monitor_status | bool | ternary('present','absent') }}" when: is_cron_installed.rc == 0 tags: - - etc-git - -- name: Cron job for /etc/.git status is installed - template: - src: etc-git-status.j2 - dest: /etc/cron.d/etc-git-status - mode: "0644" - when: is_cron_installed.rc == 0 and etc_git_monitor_status - tags: - - etc-git - -- name: Cron job for /etc/.git status is removed - file: - dest: /etc/cron.d/etc-git-status - state: absent - when: is_cron_installed.rc == 0 and not etc_git_monitor_status - tags: - - etc-git + - etc-git \ No newline at end of file diff --git a/etc-git/templates/etc-git-status.j2 b/etc-git/templates/etc-git-status.j2 deleted file mode 100644 index e1696c54..00000000 --- a/etc-git/templates/etc-git-status.j2 +++ /dev/null @@ -1,4 +0,0 @@ -# {{ ansible_managed }} - -@hourly root who > /dev/null || git --git-dir=/etc/.git --work-tree=/etc status --short -21 21 * * * root git --git-dir=/etc/.git --work-tree=/etc status --short