etc-git: manage commits with an optimized shell script instead of many slow Ansible tasks + add versioning for /usr/share/scripts

CHANGELOG

@@ -20,6 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - accounts: use "evobsd_internal_group" for SSH authentication
 - evocheck: imported version 22.03
 - base: zzz_evobackup upstream release 22.03
+- etc-git: manage commits with an optimized shell script instead of many slow Ansible tasks
+- etc-git: add versioning for /usr/share/scripts
 
 ### Fixed
 

evolixisation.yml

@@ -35,9 +35,12 @@
     # - { role: collectd, collectd_server: "127.0.0.1" }
 
   post_tasks:
-    - include: "tasks/commit_etc_git.yml"
+    - include_role:
+        name: etc-git
+        tasks_from: commit.yml
       vars:
-        commit_message: "Ansible - Evolixisation"
+        commit_message: "Ansible post-run evolisation.yml"
+
     - include_role:
         name: evocheck
         tasks_from: exec.yml

roles/etc-git/files/etc-git-optimize

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -u
+
+repositories="/etc /etc/bind/ /usr/share/scripts"
+
+for repository in ${repositories}; do
+    if [ -d "${repository}/.git" ]; then
+        git --git-dir="${repository}/.git" gc --quiet
+    fi
+done

roles/etc-git/files/etc-git-status

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -u
+
+repositories="/etc /etc/bind/ /usr/share/scripts"
+
+for repository in ${repositories}; do
+    if [ -d "${repository}/.git" ]; then
+        git --git-dir="${repository}/.git" --work-tree="${repository}" status --short
+    fi
+done

roles/etc-git/files/evocommit

@@ -0,0 +1,271 @@
+#!/bin/sh
+
+set -u
+
+VERSION="22.04"
+
+show_version() {
+    cat <<END
+evocommit version ${VERSION}
+
+Copyright 2021 Evolix <info@evolix.fr>,
+               Jérémy Lecour <jlecour@evolix.fr>
+               and others.
+
+evocommit comes with ABSOLUTELY NO WARRANTY.  This is free software,
+and you are welcome to redistribute it under certain conditions.
+See the GNU General Public Licence for details.
+END
+}
+
+show_help() {
+    cat <<END
+evocommit helps properly committing changes in a repository
+
+END
+    show_usage
+}
+show_usage() {
+    cat <<END
+Usage: evocommit --repository /path/to/repository --message "add new host"
+
+Options
+     --repository PATH   set the path for the repository
+     --message MESSAGE   set the commit message
+ -V, --version           print version number
+ -v, --verbose           increase verbosity
+ -n, --dry-run           actions are not executed
+     --help              print this message and exit
+     --version           print version and exit
+END
+}
+
+syslog() {
+    if [ -x "${LOGGER_BIN}" ]; then
+        ${LOGGER_BIN} -t "evocommit" "$1"
+    fi
+}
+get_system() {
+    uname -s
+}
+is_repository_readonly() {
+    if [ "$(get_system)" = "OpenBSD" ]; then
+        partition=$(stat -f '%Sd' $1)
+        mount | grep "${partition}" | grep -q "read-only"
+    elif command -v findmnt >/dev/null; then
+        mountpoint=$(stat -c '%m' $1)
+        findmnt "${mountpoint}" --noheadings --output OPTIONS -O ro
+    else
+        grep /usr /proc/mounts | grep -E '\bro\b'
+    fi
+}
+remount_repository_readwrite() {
+    if [ "$(get_system)" = "OpenBSD" ]; then
+        partition=$(stat -f '%Sd' $1)
+        mount -u -w /dev/${partition} 2>/dev/null
+        syslog "Re-mount ${mountpoint} as read-write to commit in repository $1"
+    else
+        mountpoint=$(stat -c '%m' $1)
+        mount -o remount,rw ${mountpoint}
+        syslog "Re-mount ${mountpoint} as read-write to commit in repository $1"
+    fi
+}
+remount_repository_readonly() {
+    if [ "$(get_system)" = "OpenBSD" ]; then
+        partition=$(stat -f '%Sd' $1)
+        mount -u -r /dev/${partition} 2>/dev/null
+        syslog "Re-mount ${mountpoint} as read-only after commit to repository $1"
+    else
+        mountpoint=$(stat -c '%m' $1)
+        mount -o remount,ro ${mountpoint} 2>/dev/null
+        syslog "Re-mount ${mountpoint} as read-only after commit to repository $1"
+    fi
+}
+is_dry_run() {
+    test "${DRY_RUN}" = "1"
+}
+is_verbose() {
+    test "${VERBOSE}" = "1"
+}
+is_ansible() {
+    test "${ANSIBLE}" = "1"
+}
+main() {
+    rc=0
+    lock="${GIT_DIR}/index.lock"
+    if [ -f "${lock}" ]; then
+        limit=$(($(date +"%s") - (1 * 60 * 60)))
+        if [ "$(get_system)" = "OpenBSD" ]; then
+            updated_at=$(stat -f "%m" "${lock}")
+        else
+            updated_at=$(stat -c "%Y" "${lock}")
+        fi
+        if [ "$updated_at" -lt "$limit" ]; then
+            rm -f "${lock}"
+        fi
+    fi
+
+    git_status=$(${GIT_BIN} status --porcelain)
+
+    if [ -n "${git_status}" ]; then
+        if is_dry_run; then
+            ${GIT_BIN} status
+        else
+            readonly_orig=0
+            # remount mount point read-write if currently readonly
+            if is_repository_readonly "${REPOSITORY}"; then
+                readonly_orig=1;
+                remount_repository_readwrite "${REPOSITORY}";
+            fi
+            author=$(logname)
+            email=$(git config --get user.email)
+            email=${email:-"${author}@evolix.net"}
+
+            # commit changes
+            git_add_result=$(${GIT_BIN} add --all)
+            git_add_rc=$?
+
+            if is_ansible; then
+                if [ ${git_add_rc} -ne 0 ]; then
+                    printf "FAILED: %s\n%s" "can't add changes in ${REPOSITORY}" "${git_add_result}"
+                    rc=1
+                fi
+            fi
+
+            git_commit_result=$(${GIT_BIN} commit --message "${MESSAGE}" --author "${author} <${email}>")
+            git_commit_rc=$?
+
+            if is_ansible; then
+                if [ ${git_commit_rc} -eq 0 ]; then
+                    printf "CHANGED: %s\n" "commit done in ${REPOSITORY} with \`${MESSAGE}'"
+                else
+                    printf "FAILED: %s\n%s" "can't commit in ${REPOSITORY} \`${MESSAGE}'" "${git_commit_result}"
+                    rc=1
+                fi
+            fi
+
+            # remount mount point read-only if it was before
+            if [ ${readonly_orig} -eq 1 ]; then
+                remount_repository_readonly "${REPOSITORY}"
+            fi
+        fi
+    else
+        if is_ansible; then
+            printf "INFO: %s\n" "no commit in ${REPOSITORY}'"
+        fi
+    fi
+
+    unset GIT_DIR
+    unset GIT_WORK_TREE
+
+    exit ${rc}
+}
+# Parse options
+# based on https://gist.github.com/deshion/10d3cb5f88a21671e17a
+while :; do
+    case ${1:-''} in
+        -h|-\?|--help)
+            show_help
+            exit 0
+            ;;
+        -V|--version)
+            show_version
+            exit 0
+            ;;
+        --message)
+            # message options, with value speparated by space
+            if [ -n "$2" ]; then
+                MESSAGE=$2
+                shift
+            else
+                printf 'ERROR: "--message" requires a non-empty option argument.\n' >&2
+                exit 1
+            fi
+            ;;
+        --message=?*)
+            # message options, with value speparated by =
+            MESSAGE=${1#*=}
+            ;;
+        --message=)
+            # message options, without value
+            printf 'ERROR: "--message" requires a non-empty option argument.\n' >&2
+            exit 1
+            ;;
+        --repository)
+            # repository options, with value speparated by space
+            if [ -n "$2" ]; then
+                REPOSITORY=$2
+                shift
+            else
+                printf 'ERROR: "--repository" requires a non-empty option argument.\n' >&2
+                exit 1
+            fi
+            ;;
+        --repository=?*)
+            # repository options, with value speparated by =
+            REPOSITORY=${1#*=}
+            ;;
+        --repository=)
+            # repository options, without value
+            printf 'ERROR: "--repository" requires a non-empty option argument.\n' >&2
+            exit 1
+            ;;
+        -n|--dry-run)
+            # disable actual commands
+            DRY_RUN=1
+            ;;
+        -v|--verbose)
+            # print verbose information
+            VERBOSE=1
+            ;;
+        --ansible)
+            # print information for Ansible
+            ANSIBLE=1
+            ;;
+        --)
+            # End of all options.
+            shift
+            break
+            ;;
+        -?*|[[:alnum:]]*)
+            # ignore unknown options
+            printf 'WARN: Unknown option (ignored): %s\n' "$1" >&2
+            ;;
+        *)
+            # Default case: If no more options then break out of the loop.
+            break
+            ;;
+    esac
+
+    shift
+done
+
+if [ -z "${MESSAGE}" ]; then
+    echo "Error: missing message parameter" >&2
+    show_usage
+    exit 1
+fi
+if [ -z "${REPOSITORY}" ]; then
+    echo "Error: missing repository parameter" >&2
+    show_usage
+    exit 1
+fi
+DRY_RUN=${DRY_RUN:-0}
+VERBOSE=${VERBOSE:-0}
+ANSIBLE=${ANSIBLE:-0}
+
+GIT_BIN=$(command -v git)
+readonly GIT_BIN
+
+LOGGER_BIN=$(command -v logger)
+readonly LOGGER_BIN
+
+export GIT_DIR="${REPOSITORY}/.git"
+export GIT_WORK_TREE="${REPOSITORY}"
+
+if [ -d "${GIT_DIR}" ]; then
+    main
+else
+    echo "There is no Git repository in '${REPOSITORY}'" >&2
+    exit 1
+fi

roles/etc-git/files/gitignore

@@ -1,4 +0,0 @@
-aliases.db
-*.swp
-random.seed
-openvpn/ipp.txt

roles/etc-git/tasks/commit.yml

@@ -1,72 +1,35 @@
 ---
-- name: is /etc clean?
-  command: git status --porcelain
-  args:
-    chdir: /etc
-  changed_when: false
-  register: git_status
-  when: not ansible_check_mode
+
+# /etc
+- name: Is /etc a git repository
+  stat:
+    path: /etc/.git
+  register: _etc_git
+
+- name: "evocommit /etc"
+  command: "/usr/local/bin/evocommit --ansible --repository /etc --message \"{{ commit_message | mandatory }}\""
+  changed_when:
+    - _etc_git_commit.stdout
+    - "'CHANGED:' in _etc_git_commit.stdout"
   ignore_errors: true
-  tags:
-    - etc-git
-    - commit-etc
+  register: _etc_git_commit
+  when:
+    - _etc_git.stat.exists
+    - _etc_git.stat.isdir
 
-- debug:
-    var: git_status
-    verbosity: 3
-  tags:
-    - etc-git
-    - commit-etc
+# /usr/share/scripts
+- name: Is /usr/share/scripts a git repository
+  stat:
+    path: /usr/share/scripts/.git
+  register: _usr_share_scripts_git
 
-- name: fetch current Git user.email
-  git_config:
-    name: user.email
-    repo: /etc
-    scope: local
-  register: git_config_user_email
+- name: "evocommit /usr/share/scripts"
+  command: "/usr/local/bin/evocommit --ansible --repository /usr/share/scripts --message \"{{ commit_message | mandatory }}\""
+  changed_when:
+    - _usr_share_scripts_git_commit.stdout
+    - "'CHANGED:' in _usr_share_scripts_git_commit.stdout"
   ignore_errors: true
-  tags:
-    - etc-git
-    - commit-etc
-
-- name: set commit author
-  set_fact:
-    commit_author: >
-      {% if ansible_env.SUDO_USER is not defined %}
-      root
-      {% else %}
-      {{ ansible_env.SUDO_USER }}
-      {% endif %}
-    commit_email: >
-      {% if git_config_user_email.config_value is not defined
-      or git_config_user_email.config_value == "" %}
-      root@localhost
-      {% else %}
-      {{ git_config_user_email.config_value }}
-      {% endif %}
-  tags:
-    - etc-git
-    - commit-etc
-
-- name: /etc modifications are committed
-  shell: >
-    git add -A .
-    && git commit
-    -m "{{ commit_message | mandatory }}"
-    --author
-    "{{ commit_author | mandatory }} <{{ commit_email | mandatory }}>"
-  args:
-    chdir: /etc
-  register: etc_commit_end_run
-  when: not ansible_check_mode and git_status.stdout != ""
-  ignore_errors: true
-  tags:
-    - etc-git
-    - commit-etc
-
-- debug:
-    var: etc_commit_end_run
-    verbosity: 4
-  tags:
-    - etc-git
-    - commit-etc
+  register: _usr_share_scripts_git_commit
+  when:
+    - _usr_share_scripts_git.stat.exists
+    - _usr_share_scripts_git.stat.isdir

roles/etc-git/tasks/main.yml

@@ -7,107 +7,72 @@
   tags:
     - etc-git
 
-- name: /etc is versioned with git
-  command: "git init ."
-  args:
-    chdir: /etc
-    creates: /etc/.git/
-    warn: false
-  register: git_init
-  tags:
-    - etc-git
-
-- name: Git user.email is configured
-  git_config:
-    name: user.email
-    repo: /etc
-    scope: local
-    value: "root@{{ inventory_hostname }}.{{ general_technical_realm }}"
-  tags:
-    - etc-git
-
-- name: /etc/.git is secure
-  file:
-    path: /etc/.git
-    owner: root
-    mode: "0700"
-    state: directory
-  tags:
-    - etc-git
-
-- name: /etc/.gitignore is present
+- name: evocommit script is installed
   copy:
-    src: gitignore
-    dest: /etc/.gitignore
-    owner: root
-    mode: "0600"
+    src: evocommit
+    dest: /usr/local/bin/evocommit
+    mode: "0755"
+    force: yes
   tags:
     - etc-git
 
-- name: Set vim as default editor
-  git_config:
-    name: core.editor
-    scope: global
-    value: vim
+- include: repository.yml
+  vars:
+    repository_path: "/etc"
+    gitignore_items:
+      - "aliases.db"
+      - "*.swp"
+      - "random.seed"
+      - "openvpn/ipp.txt"
 
-- name: does /etc/ have any commit?
-  command: "git log"
-  args:
-    chdir: /etc
-    warn: false
-  changed_when: false
-  failed_when: false
-  register: git_log
-  check_mode: false
+- name: verify /usr/share/scripts presence
+  stat:
+    path: /usr/share/scripts
+  register: _usr_share_scripts
+
+- include: repository.yml
+  vars:
+    repository_path: "/usr/share/scripts"
+    gitignore_items: []
+  when:
+    - _usr_share_scripts.stat.isdir
+
+- name: etc-git-optimize script is installed
+  copy:
+    src: etc-git-optimize
+    dest: /usr/share/scripts/etc-git-optimize
+    mode: "0755"
+    force: yes
   tags:
     - etc-git
 
-- name: initial commit is present?
-  shell: "git add -A . && git commit -m \"Initial commit via Ansible\""
-  args:
-    chdir: /etc
-    warn: false
-  register: git_commit
-  when: git_log.rc != 0 or (git_init is defined and git_init.changed)
+- name: etc-git-status script is installed
+  copy:
+    src: etc-git-status
+    dest: /usr/share/scripts/etc-git-status
+    mode: "0755"
+    force: yes
   tags:
     - etc-git
 
-- name: Optimize script is installed in monthly crontab
+- name: Legacy monthly cron job for /etc/.git optimization is absent
   lineinfile:
     path: /etc/monthly.local
     line: '/usr/local/bin/git --git-dir /etc/.git gc --quiet'
-    owner: root
-    mode: "0644"
-    create: true
+    state: absent
   tags:
     - etc-git
 
-- name: cron job for /etc/.git status is installed
-  lineinfile:
-    path: /etc/daily.local
-    line:
-      '/usr/local/bin/git --git-dir=/etc/.git --work-tree=/etc status --short'
-    owner: root
-    mode: "0644"
-    create: true
-  when: etc_git_monitor_status
+- name: Legacy hourly cron job for /etc/.git status is absent
+  cron:
+    name: git status
+    minute: 42
+    job: who > /dev/null || /usr/local/bin/git --git-dir=/etc/.git --work-tree=/etc status --short
+    state: absent
   tags:
     - etc-git
 
-- name: cron job for /etc/.git status is installed - next_part
-  lineinfile:
-    path: /etc/daily.local
-    line: 'next_part "Checking /etc git status:"'
-    insertbefore:
-      '/usr/local/bin/git --git-dir=/etc/.git --work-tree=/etc status --short'
-    owner: root
-    mode: "0644"
-    create: true
-  when: etc_git_monitor_status
-  tags:
-    - etc-git
-
-- name: cron job for /etc/.git status is removed
+- name: Legacy daily cron jobs for /etc/.git status are absent
   lineinfile:
     path: /etc/daily.local
     line: "{{ item }}"
@@ -117,37 +82,72 @@
   with_items:
     - 'next_part "Checking /etc git status:"'
     - '/usr/local/bin/git --git-dir=/etc/.git --work-tree=/etc status --short'
-  when: not etc_git_monitor_status
   tags:
     - etc-git
 
-- name: hourly cron job for /etc/.git status is installed
-  cron:
-    name: git status
-    minute: "42"
-    job: >
-      who
-      > /dev/null
-      || /usr/local/bin/git
-      --git-dir=/etc/.git
-      --work-tree=/etc
-      status --short
-  when: etc_git_monitor_status
+- name: Cron job for monthly git optimization
+  lineinfile:
+    path: /etc/monthly.local
+    line: "/usr/share/scripts/etc-git-optimize"
+    owner: root
+    mode: "0644"
+    create: true
   tags:
     - etc-git
 
-- name: hourly cron job for /etc/.git status is removed
-  cron:
-    name: git status
-    minute: 42
-    job: >
-      who
-      > /dev/null
-      || /usr/local/bin/git
-      --git-dir=/etc/.git
-      --work-tree=/etc
-      status --short
-    state: absent
-  when: not etc_git_monitor_status
+- name: Cron job for monthly git optimization - next_part
+  lineinfile:
+    path: /etc/monthly.local
+    line: 'next_part "Monthly optimization:"'
+    insertbefore: "/usr/share/scripts/etc-git-optimize"
+    owner: root
+    mode: "0644"
+    create: true
+  tags:
+    - etc-git
+
+- name: Cron job for hourly git status
+  lineinfile:
+    path: /etc/hourly.local
+    line: "who > /dev/null || /usr/share/scripts/etc-git-status"
+    owner: root
+    mode: "0644"
+    create: true
+    state: "{{ etc_git_monitor_status | bool | ternary('present','absent') }}"
+  tags:
+    - etc-git
+
+- name: Cron job for hourly git status - next_part
+  lineinfile:
+    path: /etc/hourly.local
+    line: 'next_part "Hourly warning for unclean Git repository if nobody is connected:"'
+    insertbefore: "who > /dev/null || /usr/share/scripts/etc-git-status"
+    owner: root
+    mode: "0644"
+    create: true
+    state: "{{ etc_git_monitor_status | bool | ternary('present','absent') }}"
+  tags:
+    - etc-git
+
+- name: Cron job for daily git status
+  lineinfile:
+    path: /etc/daily.local
+    line: "/usr/share/scripts/etc-git-status"
+    owner: root
+    mode: "0644"
+    create: true
+    state: "{{ etc_git_monitor_status | bool | ternary('present','absent') }}"
+  tags:
+    - etc-git
+
+- name: Cron job for daily git status - next_part
+  lineinfile:
+    path: /etc/daily.local
+    line: 'next_part "Daily warning for unclean Git repository:"'
+    insertbefore: "/usr/share/scripts/etc-git-status"
+    owner: root
+    mode: "0644"
+    create: true 
+    state: "{{ etc_git_monitor_status | bool | ternary('present','absent') }}"
   tags:
     - etc-git

roles/etc-git/tasks/repository.yml

@@ -0,0 +1,75 @@
+---
+
+- name: "{{ repository_path }} is versioned with git"
+  command: "git init ."
+  args:
+    chdir: "{{ repository_path }}"
+    creates: "{{ repository_path }}/.git/"
+    warn: false
+  register: git_init
+  tags:
+    - etc-git
+
+- name: Git user.email is configured
+  git_config:
+    name: user.email
+    repo: "{{ repository_path }}"
+    scope: local
+    value: "root@{{ inventory_hostname }}.{{ general_technical_realm }}"
+  tags:
+    - etc-git
+
+- name: "{{ repository_path }}/.git is restricted to root"
+  file:
+    path: "{{ repository_path }}/.git"
+    owner: root
+    mode: "0700"
+    state: directory
+  tags:
+    - etc-git
+
+- name: "{{ repository_path }}/.gitignore is present"
+  file:
+    path: "{{ repository_path }}/.gitignore"
+    owner: root
+    mode: "0600"
+  tags:
+    - etc-git
+
+- name: "Some entries MUST be in the {{ repository_path }}/.gitignore file"
+  lineinfile:
+    dest: "{{ repository_path }}/.gitignore"
+    line: "{{ item }}"
+  loop: "{{ gitignore_items | default([]) }}"
+  tags:
+    - etc-git
+
+- name: Set vim as default editor
+  git_config:
+    name: core.editor
+    scope: global
+    value: vim
+  tags:
+    - etc-git
+
+- name: "does {{ repository_path }}/ have any commit?"
+  command: "git log"
+  args:
+    chdir: "{{ repository_path }}"
+    warn: false
+  changed_when: false
+  failed_when: false
+  register: git_log
+  check_mode: false
+  tags:
+    - etc-git
+
+- name: initial commit is present?
+  shell: "git add -A . && git commit -m \"Initial commit via Ansible\""
+  args:
+    chdir: "{{ repository_path }}"
+    warn: false
+  register: git_commit
+  when: git_log.rc != 0 or (git_init is defined and git_init is changed)
+  tags:
+    - etc-git

tasks/commit_etc_git.yml

@@ -1,27 +0,0 @@
----
-- name: is /etc clean?
-  command: git status --porcelain
-  args:
-    chdir: /etc
-  changed_when: false
-  register: git_status
-  when: not ansible_check_mode
-  ignore_errors: true
-  tags:
-    - commit-etc
-# yamllint disable rule:line-length
-- name: /etc modifications are committed
-  shell: >
-    git add -A .
-    && git commit
-    -m "{{ commit_message | default('Ansible run') }}"
-    --author="{{ ansible_env.SUDO_USER | default('Root') }}
-    <{{ ansible_env.SUDO_USER | default('Root') }}@{{ general_technical_realm }}>"
-  args:
-    chdir: /etc
-  register: etc_commit_end_evolinux
-  when: not ansible_check_mode and git_status.stdout != ""
-  ignore_errors: true
-  tags:
-    - commit-etc
-# yamllint enable rule:line-length