From 367bda528fc76383907633e00860f1e8009238b6 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Fri, 17 Mar 2023 20:05:42 +0100 Subject: [PATCH] apt: use deb822 format on Debian 12 --- apt/files/deb822-migration.py | 122 +++++++++++++----- apt/files/deb822-migration.sh | 61 ++++----- apt/tasks/backports.deb822.yml | 35 +++++ .../{backports.yml => backports.oneline.yml} | 4 +- apt/tasks/basics.deb822.yml | 28 ++++ apt/tasks/basics.oneline.yml | 18 +++ apt/tasks/basics.yml | 33 ----- apt/tasks/evolix_public.deb822.yml | 45 +++++++ ...x_public.yml => evolix_public.oneline.yml} | 4 +- apt/tasks/hold_packages.yml | 2 +- apt/tasks/main.yml | 90 +++++++++++-- apt/templates/bookworm_backports.sources.j2 | 7 + apt/templates/bookworm_basics.list.j2 | 5 - apt/templates/bookworm_basics.sources.j2 | 9 ++ apt/templates/bookworm_security.sources.j2 | 7 + 15 files changed, 351 insertions(+), 119 deletions(-) mode change 100644 => 100755 apt/files/deb822-migration.py mode change 100644 => 100755 apt/files/deb822-migration.sh create mode 100644 apt/tasks/backports.deb822.yml rename apt/tasks/{backports.yml => backports.oneline.yml} (100%) create mode 100644 apt/tasks/basics.deb822.yml create mode 100644 apt/tasks/basics.oneline.yml delete mode 100644 apt/tasks/basics.yml create mode 100644 apt/tasks/evolix_public.deb822.yml rename apt/tasks/{evolix_public.yml => evolix_public.oneline.yml} (100%) create mode 100644 apt/templates/bookworm_backports.sources.j2 delete mode 100644 apt/templates/bookworm_basics.list.j2 create mode 100644 apt/templates/bookworm_basics.sources.j2 create mode 100644 apt/templates/bookworm_security.sources.j2 diff --git a/apt/files/deb822-migration.py b/apt/files/deb822-migration.py old mode 100644 new mode 100755 index 10ee47ae..a8873923 --- a/apt/files/deb822-migration.py +++ b/apt/files/deb822-migration.py @@ -3,20 +3,36 @@ import re import sys import os +import select +import apt +import apt_pkg -if len(sys.argv) > 1: - src_file = sys.argv[1] -else: - print("You must provide a source file as first argument", file=sys.stderr) - sys.exit(1) +# Order matters ! +destinations = { + "debian-security": "security.sources", + ".*-backports": "backports.sources", + ".debian.org": "system.sources", + "mirror.evolix.org": "system.sources", + "pub.evolix.net": "evolix_public_old.sources", + "pub.evolix.org": "evolix_public.sources", + "artifacts.elastic.co": "elastic.sources", + "download.docker.com": "docker.sources", + "downloads.linux.hpe.com": "hp.sources", + "pkg.jenkins-ci.org": "jenkins.sources", + "packages.sury.org": "sury.sources", + "repo.mongodb.org": "mongodb.sources", + "apt.newrelic.com": "newrelic.sources", + "deb.nodesource.com": "nodesource.sources", + "dl.yarnpkg.com": "yarn.sources", + "apt.postgresql.org": "postgresql.sources", + "packages.microsoft.com/repos/vscode": "microsoft-vscode.sources", + "packages.microsoft.com/repos/ms-teams": "microsoft-teams.sources", + "updates.signal.org": "signal.sources", + "downloads.1password.com/linux/debian": "1password.sources", + "download.virtualbox.org": "virtualbox.sources" +} -if not os.access(src_file, os.R_OK): - print(src_file, "is not readable", file=sys.stderr) - sys.exit(2) - -pattern = re.compile('^(?Pdeb|deb-src) +(?P\[.+\] ?)*(?P\w+:\/\/\S+) +(?P\S+)(?: +(?P.*))?$') - -sources = {} +sources_parts = apt_pkg.config.find_dir('Dir::Etc::sourceparts') def split_options(raw): table = str.maketrans({ @@ -27,25 +43,44 @@ def split_options(raw): return options -with open(src_file,'r') as file: - for line in file: - matches = re.match(pattern, line) - if matches is not None: - # print(matches.groupdict()) - uri = matches['uri'] +def auto_destination(uri): + basename = uri + basename = re.sub('\[[^\]]+\]', '', basename) + basename = re.sub('\w+://', '', basename) + basename = '_'.join(re.sub('[^a-zA-Z0-9]', ' ', basename).split()) + return '%s.sources' % basename + +def destination(matches): + for search_str in destinations.keys(): + search_pattern = re.compile(f'{search_str}(/|\s|$)') + if re.search(search_pattern, matches['uri']) or re.search(search_pattern, matches["suite"]): + return destinations[search_str] + # fallback if nothing matches + return auto_destination(matches['uri']) + +def prepare_sources(lines): + sources = {} + pattern = re.compile('^(?: *(?Pdeb|deb-src)) +(?P\[.+\] ?)*(?P\w+:\/\/\S+) +(?P\S+)(?: +(?P.*))?$') + + for line in lines: + matches = re.match(pattern, line) + + if matches is not None: + dest = destination(matches) options = {} + if matches.group('options'): for option in split_options(matches['options']): if "=" in option: key, value = option.split("=") options[key] = value - if uri in sources: - sources[uri]["Types"].add(matches["type"]) - sources[uri]["URIs"] = matches["uri"] - sources[uri]["Suites"].add(matches["suite"]) - sources[uri]["Components"].update(matches["components"].split(' ')) + if dest in sources: + sources[dest]["Types"].add(matches["type"]) + sources[dest]["URIs"] = matches["uri"] + sources[dest]["Suites"].add(matches["suite"]) + sources[dest]["Components"].update(matches["components"].split(' ')) else: source = { "Types": {matches['type']}, @@ -83,14 +118,35 @@ with open(src_file,'r') as file: else: source["Targets"] = {options["target"]} - sources[uri] = source + sources[dest] = source + return sources -for i, (uri, source) in enumerate(sources.items()): - if i > 0: - print("") - for key, value in source.items(): - if isinstance(value, str): - print("{}: {}".format(key, value) ) - else: - print("{}: {}".format(key, ' '.join(value)) ) - i += 1 \ No newline at end of file +def save_sources(sources, output_dir): + # print(output_dir) + # print(sources) + for dest, source in sources.items(): + source_path = output_dir + dest + + with open(source_path, 'w') as file: + for key, value in source.items(): + if isinstance(value, str): + file.write("{}: {}\n".format(key, value)) + else: + file.write("{}: {}\n".format(key, ' '.join(value))) + +def main(): + if select.select([sys.stdin, ], [], [], 0.0)[0]: + sources = prepare_sources(sys.stdin) + # elif len(sys.argv) > 1: + # sources = prepare_sources([sys.argv[1]]) + else: + print("You must provide source lines to stdin", file=sys.stderr) + sys.exit(1) + + output_dir = apt_pkg.config.find_dir('Dir::Etc::sourceparts') + save_sources(sources, output_dir) + +if __name__ == "__main__": + main() + +sys.exit(0) \ No newline at end of file diff --git a/apt/files/deb822-migration.sh b/apt/files/deb822-migration.sh old mode 100644 new mode 100755 index 4e4a4dbc..10fb7889 --- a/apt/files/deb822-migration.sh +++ b/apt/files/deb822-migration.sh @@ -10,39 +10,40 @@ if [ ! -x "${deb822_migrate_script}" ]; then exit 1 fi -dest_dir="/etc/apt/sources.list.d" -rc=0 - -migrate_file() { - legacy_file=$1 - deb822_file=$2 - - if [ -f "${legacy_file}" ]; then - if [ -f "${deb822_file}" ]; then - >&2 echo "ERROR: '${deb822_file}' already exists" - rc=2 - else - ${deb822_migrate_script} "${legacy_file}" > "${deb822_file}" - if [ $? -eq 0 ] && [ -f "${deb822_file}" ]; then - mv "${legacy_file}" "${legacy_file}.bak" - echo "Migrated ${legacy_file} to ${deb822_file} and renamed to ${legacy_file}.bak" - else - >&2 echo "ERROR: failed to convert '${legacy_file}' to '${deb822_file}'" - rc=2 - fi - fi - else - >&2 echo "ERROR: '${legacy_file}' not found" - rc=2 - fi +sources_from_file() { + grep --extended-regexp "^\s*(deb|deb-src) " $1 } -migrate_file "/etc/apt/sources.list" "${dest_dir}/system.sources" +rc=0 +count=0 -# shellcheck disable=SC2044 -for legacy_file in $(find /etc/apt/sources.list.d -mindepth 1 -maxdepth 1 -type f -name '*.list'); do - deb822_file=$(basename "${legacy_file}" .list) - migrate_file "${legacy_file}" "${dest_dir}/${deb822_file}.sources" +if [ -f /etc/apt/sources.list ]; then + sources_from_file /etc/apt/sources.list | ${deb822_migrate_script} + python_rc=$? + + if [ ${python_rc} -eq 0 ]; then + mv /etc/apt/sources.list /etc/apt/sources.list.bak + echo "OK: /etc/apt/sources.list" + count=$(( count + 1 )) + else + >&2 echo "ERROR: failed migration for /etc/apt/sources.list" + rc=1 + fi +fi + +for file in $(find /etc/apt/sources.list.d -mindepth 1 -maxdepth 1 -type f -name '*.list'); do + sources_from_file "${file}" | ${deb822_migrate_script} + python_rc=$? + + if [ ${python_rc} -eq 0 ]; then + mv "${file}" "${file}.bak" + echo "OK: ${file}" + count=$(( count + 1 )) + else + >&2 echo "ERROR: failed migration for ${file}" + rc=1 + fi done +echo "${count} file(s) migrated" exit ${rc} \ No newline at end of file diff --git a/apt/tasks/backports.deb822.yml b/apt/tasks/backports.deb822.yml new file mode 100644 index 00000000..8e196cc0 --- /dev/null +++ b/apt/tasks/backports.deb822.yml @@ -0,0 +1,35 @@ +--- +- name: No backports config in default sources.list + lineinfile: + dest: /etc/apt/sources.list.d/ + regexp: "backports" + state: absent + tags: + - apt + +- name: Backports sources list is installed + template: + src: '{{ ansible_distribution_release }}_backports.sources.j2' + dest: /etc/apt/sources.list.d/backports.sources + force: yes + mode: "0640" + register: apt_backports_sources + tags: + - apt + +- name: Backports configuration + copy: + src: '{{ ansible_distribution_release }}_backports_preferences' + dest: /etc/apt/preferences.d/0-backports-defaults + force: yes + mode: "0640" + register: apt_backports_config + tags: + - apt + +- name: Apt update + apt: + update_cache: yes + when: apt_backports_sources is changed or apt_backports_config is changed + tags: + - apt diff --git a/apt/tasks/backports.yml b/apt/tasks/backports.oneline.yml similarity index 100% rename from apt/tasks/backports.yml rename to apt/tasks/backports.oneline.yml index aecf6194..7f6509b0 100644 --- a/apt/tasks/backports.yml +++ b/apt/tasks/backports.oneline.yml @@ -33,13 +33,13 @@ line: 'Acquire::Check-Valid-Until no;' create: yes state: present - when: ansible_distribution_release == "jessie" tags: - apt + when: ansible_distribution_release == "jessie" - name: Apt update apt: update_cache: yes - when: apt_backports_list is changed or apt_backports_config is changed tags: - apt + when: apt_backports_list is changed or apt_backports_config is changed diff --git a/apt/tasks/basics.deb822.yml b/apt/tasks/basics.deb822.yml new file mode 100644 index 00000000..0a342e61 --- /dev/null +++ b/apt/tasks/basics.deb822.yml @@ -0,0 +1,28 @@ +--- + +- name: Change basics repositories + template: + src: "{{ ansible_distribution_release }}_basics.sources.j2" + dest: /etc/apt/sources.list.d/system.sources + mode: "0644" + force: yes + register: apt_basic_sources + tags: + - apt + +- name: Change security repositories + template: + src: "{{ ansible_distribution_release }}_security.sources.j2" + dest: /etc/apt/sources.list.d/security.sources + mode: "0644" + force: yes + register: apt_security_sources + tags: + - apt + +- name: Apt update + apt: + update_cache: yes + tags: + - apt + when: apt_basic_list is changed or apt_security_sources is changed diff --git a/apt/tasks/basics.oneline.yml b/apt/tasks/basics.oneline.yml new file mode 100644 index 00000000..8e0a562c --- /dev/null +++ b/apt/tasks/basics.oneline.yml @@ -0,0 +1,18 @@ +--- + +- name: Change basics repositories + template: + src: "{{ ansible_distribution_release }}_basics.list.j2" + dest: /etc/apt/sources.list + mode: "0644" + force: yes + register: apt_basic_list + tags: + - apt + +- name: Apt update + apt: + update_cache: yes + tags: + - apt + when: apt_basic_list is changed diff --git a/apt/tasks/basics.yml b/apt/tasks/basics.yml deleted file mode 100644 index 33c79129..00000000 --- a/apt/tasks/basics.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- - -- name: Change basics repositories - template: - src: "{{ ansible_distribution_release }}_basics.list.j2" - dest: /etc/apt/sources.list - mode: "0644" - force: yes - register: apt_basic_list - tags: - - apt - -- name: Clean GANDI sources.list.d/debian-security.list - file: - path: '{{ item }}' - state: absent - loop: - - /etc/apt/sources.list.d/debian-security.list - - /etc/apt/sources.list.d/debian-jessie.list - - /etc/apt/sources.list.d/debian-stretch.list - - /etc/apt/sources.list.d/debian-buster.list - - /etc/apt/sources.list.d/debian-bullseye.list - - /etc/apt/sources.list.d/debian-update.list - when: apt_clean_gandi_sourceslist | bool - tags: - - apt - -- name: Apt update - apt: - update_cache: yes - when: apt_basic_list is changed - tags: - - apt diff --git a/apt/tasks/evolix_public.deb822.yml b/apt/tasks/evolix_public.deb822.yml new file mode 100644 index 00000000..a98a9983 --- /dev/null +++ b/apt/tasks/evolix_public.deb822.yml @@ -0,0 +1,45 @@ +--- + +- name: Look for legacy apt keyring + stat: + path: /etc/apt/trusted.gpg + register: _trusted_gpg_keyring + tags: + - apt + +- name: Evolix embedded GPG key is absent + apt_key: + id: "B8612B5D" + keyring: /etc/apt/trusted.gpg + state: absent + tags: + - apt + when: _trusted_gpg_keyring.stat.exists + +- name: Add Evolix GPG key + copy: + src: pub_evolix.asc + dest: "{{ apt_keyring_dir }}/pub_evolix.asc" + force: yes + mode: "0644" + owner: root + group: root + tags: + - apt + +- name: Evolix public list is installed + template: + src: evolix_public.sources.j2 + dest: /etc/apt/sources.list.d/evolix_public.sources + force: yes + mode: "0640" + register: apt_evolix_public + tags: + - apt + +- name: Apt update + apt: + update_cache: yes + tags: + - apt + when: apt_evolix_public is changed diff --git a/apt/tasks/evolix_public.yml b/apt/tasks/evolix_public.oneline.yml similarity index 100% rename from apt/tasks/evolix_public.yml rename to apt/tasks/evolix_public.oneline.yml index 8c4d5216..e3ca833e 100644 --- a/apt/tasks/evolix_public.yml +++ b/apt/tasks/evolix_public.oneline.yml @@ -12,9 +12,9 @@ id: "B8612B5D" keyring: /etc/apt/trusted.gpg state: absent - when: _trusted_gpg_keyring.stat.exists tags: - apt + when: _trusted_gpg_keyring.stat.exists - name: Add Evolix GPG key copy: @@ -40,6 +40,6 @@ - name: Apt update apt: update_cache: yes - when: apt_evolix_public is changed tags: - apt + when: apt_evolix_public is changed diff --git a/apt/tasks/hold_packages.yml b/apt/tasks/hold_packages.yml index 10f5b358..2b3b815f 100644 --- a/apt/tasks/hold_packages.yml +++ b/apt/tasks/hold_packages.yml @@ -97,6 +97,6 @@ day: "{{ apt_check_hold_cron_day }}" month: "{{ apt_check_hold_cron_month }}" state: "present" - when: is_cron.rc == 0 tags: - apt + when: is_cron.rc == 0 diff --git a/apt/tasks/main.yml b/apt/tasks/main.yml index 353dca36..3459b1b5 100644 --- a/apt/tasks/main.yml +++ b/apt/tasks/main.yml @@ -1,10 +1,26 @@ --- - name: "Compatibility check" - fail: - msg: only compatible with Debian >= 8 - when: - - ansible_distribution != "Debian" or ansible_distribution_major_version is version('8', '<') + assert: + that: + - ansible_distribution = "Debian" + - ansible_distribution_major_version is version('8', '>=') + msg: Only compatible with Debian >= 8 + tags: + - apt + +- name: "apt-transport-https is installed for https repositories (before Buster)" + apt: + name: + - apt-transport-https + tags: + - apt + when: ansible_distribution_major_version is version('10', '<') + +- name: "certificates are installed to https repositories" + apt: + name: + - ca-certificates tags: - apt @@ -14,23 +30,71 @@ tags: - apt -- name: Install basics repositories - include: basics.yml - when: apt_install_basics | bool +- name: Install basics repositories (Debian <12) + include: basics.debian-lt-12.yml tags: - apt + when: + - apt_install_basics | bool + - ansible_distribution_major_version is version('12', '<') -- name: Install APT Backports repository - include: backports.yml - when: apt_install_backports | bool +- name: Install basics repositories (Debian >=12) + include: basics.debian-ge-12.yml tags: - apt + when: + - apt_install_basics | bool + - ansible_distribution_major_version is version('12', '>=') -- name: Install Evolix Public APT repository - include: evolix_public.yml - when: apt_install_evolix_public | bool + +- name: Install backports repositories (Debian <12) + include: backports.debian-lt-12.yml tags: - apt + when: + - apt_install_backports | bool + - ansible_distribution_major_version is version('12', '<') + +- name: Install backports repositories (Debian >=12) + include: backports.debian-ge-12.yml + tags: + - apt + when: + - apt_install_backports | bool + - ansible_distribution_major_version is version('12', '>=') + + +- name: Install Evolix Public repositories (Debian <12) + include: evolix_public.debian-lt-12.yml + tags: + - apt + when: + - apt_install_evolix_public | bool + - ansible_distribution_major_version is version('12', '<') + +- name: Install Evolix Public repositories (Debian >=12) + include: evolix_public.debian-ge-12.yml + tags: + - apt + when: + - apt_install_evolix_public | bool + - ansible_distribution_major_version is version('12', '>=') + +- name: Clean GANDI sources + file: + path: '{{ item }}' + state: absent + loop: + - /etc/apt/sources.list.d/debian-security.list + - /etc/apt/sources.list.d/debian-jessie.list + - /etc/apt/sources.list.d/debian-stretch.list + - /etc/apt/sources.list.d/debian-buster.list + - /etc/apt/sources.list.d/debian-bullseye.list + - /etc/apt/sources.list.d/debian-update.list + tags: + - apt + when: apt_clean_gandi_sourceslist | bool + - name: Install check for packages marked hold include: hold_packages.yml diff --git a/apt/templates/bookworm_backports.sources.j2 b/apt/templates/bookworm_backports.sources.j2 new file mode 100644 index 00000000..20a505a3 --- /dev/null +++ b/apt/templates/bookworm_backports.sources.j2 @@ -0,0 +1,7 @@ +# {{ ansible_managed }} + +Types: deb +URIs: https://mirror.evolix.org/debian +Suites: bullseye-backports +Components: {{ apt_backports_components | mandatory }} +Enabled: yes diff --git a/apt/templates/bookworm_basics.list.j2 b/apt/templates/bookworm_basics.list.j2 deleted file mode 100644 index 1c6bc15b..00000000 --- a/apt/templates/bookworm_basics.list.j2 +++ /dev/null @@ -1,5 +0,0 @@ -# {{ ansible_managed }} - -deb http://mirror.evolix.org/debian bookworm {{ apt_basics_components | mandatory }} -deb http://mirror.evolix.org/debian/ bookworm-updates {{ apt_basics_components | mandatory }} -deb http://security.debian.org/debian-security bookworm-security {{ apt_basics_components | mandatory }} diff --git a/apt/templates/bookworm_basics.sources.j2 b/apt/templates/bookworm_basics.sources.j2 new file mode 100644 index 00000000..fbc3034a --- /dev/null +++ b/apt/templates/bookworm_basics.sources.j2 @@ -0,0 +1,9 @@ +# {{ ansible_managed }} + +Types: deb +URIs: https://mirror.evolix.org/debian +Suites: bookworm bookworm-updates +Components: {{ apt_basics_components | mandatory }} +Enabled: yes + +deb http://security.debian.org/debian-security bookworm-security {{ apt_basics_components | mandatory }} diff --git a/apt/templates/bookworm_security.sources.j2 b/apt/templates/bookworm_security.sources.j2 new file mode 100644 index 00000000..0b0e4190 --- /dev/null +++ b/apt/templates/bookworm_security.sources.j2 @@ -0,0 +1,7 @@ +# {{ ansible_managed }} + +Types: deb +URIs: https://security.debian.org/debian-security +Suites: bookworm-security +Components: {{ apt_basics_components | mandatory }} +Enabled: yes \ No newline at end of file