apt: use deb822 format on Debian 12

apt/files/deb822-migration.py

@@ -3,20 +3,36 @@
 import re
 import sys
 import os
+import select
+import apt
+import apt_pkg
 
-if len(sys.argv) > 1:
-    src_file = sys.argv[1]
-else:
-    print("You must provide a source file as first argument", file=sys.stderr)
-    sys.exit(1)
+# Order matters !
+destinations = {
+    "debian-security": "security.sources",
+    ".*-backports": "backports.sources",
+    ".debian.org": "system.sources",
+    "mirror.evolix.org": "system.sources",
+    "pub.evolix.net": "evolix_public_old.sources",
+    "pub.evolix.org": "evolix_public.sources",
+    "artifacts.elastic.co": "elastic.sources",
+    "download.docker.com": "docker.sources",
+    "downloads.linux.hpe.com": "hp.sources",
+    "pkg.jenkins-ci.org": "jenkins.sources",
+    "packages.sury.org": "sury.sources",
+    "repo.mongodb.org": "mongodb.sources",
+    "apt.newrelic.com": "newrelic.sources",
+    "deb.nodesource.com": "nodesource.sources",
+    "dl.yarnpkg.com": "yarn.sources",
+    "apt.postgresql.org": "postgresql.sources",
+    "packages.microsoft.com/repos/vscode": "microsoft-vscode.sources",
+    "packages.microsoft.com/repos/ms-teams": "microsoft-teams.sources",
+    "updates.signal.org": "signal.sources",
+    "downloads.1password.com/linux/debian": "1password.sources",
+    "download.virtualbox.org": "virtualbox.sources"
+}
 
-if not os.access(src_file, os.R_OK):
-    print(src_file, "is not readable", file=sys.stderr)
-    sys.exit(2)
-
-pattern = re.compile('^(?P<type>deb|deb-src) +(?P<options>\[.+\] ?)*(?P<uri>\w+:\/\/\S+) +(?P<suite>\S+)(?: +(?P<components>.*))?$')
-
-sources = {}
+sources_parts = apt_pkg.config.find_dir('Dir::Etc::sourceparts')
 
 def split_options(raw):
     table = str.maketrans({
@@ -27,25 +43,44 @@ def split_options(raw):
 
     return options
 
-with open(src_file,'r') as file:
-    for line in file:
-        matches = re.match(pattern, line)
-        if matches is not None:
-            # print(matches.groupdict())
-            uri = matches['uri']
+def auto_destination(uri):
+    basename = uri
+    basename = re.sub('\[[^\]]+\]', '', basename)
+    basename = re.sub('\w+://', '', basename)
+    basename = '_'.join(re.sub('[^a-zA-Z0-9]', ' ', basename).split())
+    return '%s.sources' % basename
 
+
+def destination(matches):
+    for search_str in destinations.keys():
+        search_pattern = re.compile(f'{search_str}(/|\s|$)')
+        if re.search(search_pattern, matches['uri']) or re.search(search_pattern, matches["suite"]):
+            return destinations[search_str]
+    # fallback if nothing matches
+    return auto_destination(matches['uri'])
+
+def prepare_sources(lines):
+    sources = {}
+    pattern = re.compile('^(?: *(?P<type>deb|deb-src)) +(?P<options>\[.+\] ?)*(?P<uri>\w+:\/\/\S+) +(?P<suite>\S+)(?: +(?P<components>.*))?$')
+
+    for line in lines:
+        matches = re.match(pattern, line)
+
+        if matches is not None:
+            dest = destination(matches)
             options = {}
+
             if matches.group('options'):
                 for option in split_options(matches['options']):
                     if "=" in option:
                         key, value = option.split("=")
                         options[key] = value
 
-            if uri in sources:
-                sources[uri]["Types"].add(matches["type"])
-                sources[uri]["URIs"] = matches["uri"]
-                sources[uri]["Suites"].add(matches["suite"])
-                sources[uri]["Components"].update(matches["components"].split(' '))
+            if dest in sources:
+                sources[dest]["Types"].add(matches["type"])
+                sources[dest]["URIs"] = matches["uri"]
+                sources[dest]["Suites"].add(matches["suite"])
+                sources[dest]["Components"].update(matches["components"].split(' '))
             else:
                 source = {
                     "Types": {matches['type']},
@@ -83,14 +118,35 @@ with open(src_file,'r') as file:
                     else:
                         source["Targets"] = {options["target"]}
 
-                sources[uri] = source
+                sources[dest] = source
+    return sources
 
-for i, (uri, source) in enumerate(sources.items()):
-    if i > 0:
-        print("")
-    for key, value in source.items():
-        if isinstance(value, str):
-            print("{}: {}".format(key, value) )
-        else:
-            print("{}: {}".format(key, ' '.join(value)) )
-    i += 1
+def save_sources(sources, output_dir):
+    # print(output_dir)
+    # print(sources)
+    for dest, source in sources.items():
+        source_path = output_dir + dest
+
+        with open(source_path, 'w') as file:
+            for key, value in source.items():
+                if isinstance(value, str):
+                    file.write("{}: {}\n".format(key, value))
+                else:
+                    file.write("{}: {}\n".format(key, ' '.join(value)))
+
+def main():
+    if select.select([sys.stdin, ], [], [], 0.0)[0]:
+        sources = prepare_sources(sys.stdin)
+    # elif len(sys.argv) > 1:
+    #     sources = prepare_sources([sys.argv[1]])
+    else:
+        print("You must provide source lines to stdin", file=sys.stderr)
+        sys.exit(1)
+
+    output_dir = apt_pkg.config.find_dir('Dir::Etc::sourceparts')
+    save_sources(sources, output_dir)
+
+if __name__ == "__main__":
+    main()
+
+sys.exit(0)

apt/files/deb822-migration.sh

@@ -10,39 +10,40 @@ if [ ! -x "${deb822_migrate_script}" ]; then
     exit 1
 fi
 
-dest_dir="/etc/apt/sources.list.d"
-rc=0
-
-migrate_file() {
-    legacy_file=$1
-    deb822_file=$2
-
-    if [ -f "${legacy_file}" ]; then
-        if [ -f "${deb822_file}" ]; then
-            >&2 echo "ERROR: '${deb822_file}' already exists"
-            rc=2
-        else
-            ${deb822_migrate_script} "${legacy_file}" > "${deb822_file}"
-            if [ $? -eq 0 ] && [ -f "${deb822_file}" ]; then
-                mv "${legacy_file}" "${legacy_file}.bak"
-                echo "Migrated ${legacy_file} to ${deb822_file} and renamed to ${legacy_file}.bak"
-            else
-                >&2 echo "ERROR: failed to convert '${legacy_file}' to '${deb822_file}'"
-                rc=2
-            fi
-        fi
-    else
-        >&2 echo "ERROR: '${legacy_file}' not found"
-        rc=2
-    fi
+sources_from_file() {
+    grep --extended-regexp "^\s*(deb|deb-src) " $1
 }
 
-migrate_file "/etc/apt/sources.list" "${dest_dir}/system.sources"
+rc=0
+count=0
 
-# shellcheck disable=SC2044
-for legacy_file in $(find /etc/apt/sources.list.d -mindepth 1 -maxdepth 1 -type f -name '*.list'); do
-    deb822_file=$(basename "${legacy_file}" .list)
-    migrate_file "${legacy_file}" "${dest_dir}/${deb822_file}.sources"
+if [ -f /etc/apt/sources.list ]; then
+    sources_from_file /etc/apt/sources.list | ${deb822_migrate_script} 
+    python_rc=$?
+
+    if [ ${python_rc} -eq 0 ]; then
+        mv /etc/apt/sources.list /etc/apt/sources.list.bak
+        echo "OK: /etc/apt/sources.list"
+        count=$(( count + 1 ))
+    else
+        >&2 echo "ERROR: failed migration for /etc/apt/sources.list"
+        rc=1
+    fi
+fi
+
+for file in $(find /etc/apt/sources.list.d -mindepth 1 -maxdepth 1 -type f -name '*.list'); do
+    sources_from_file "${file}" | ${deb822_migrate_script}
+    python_rc=$?
+
+    if [ ${python_rc} -eq 0 ]; then
+        mv "${file}" "${file}.bak"
+        echo "OK: ${file}"
+        count=$(( count + 1 ))
+    else
+        >&2 echo "ERROR: failed migration for ${file}"
+        rc=1
+    fi
 done
 
+echo "${count} file(s) migrated"
 exit ${rc}

apt/tasks/backports.deb822.yml

@@ -0,0 +1,35 @@
+---
+- name: No backports config in default sources.list
+  lineinfile:
+    dest: /etc/apt/sources.list.d/
+    regexp: "backports"
+    state: absent
+  tags:
+    - apt
+
+- name: Backports sources list is installed
+  template:
+    src: '{{ ansible_distribution_release }}_backports.sources.j2'
+    dest: /etc/apt/sources.list.d/backports.sources
+    force: yes
+    mode: "0640"
+  register: apt_backports_sources
+  tags:
+    - apt
+
+- name: Backports configuration
+  copy:
+    src: '{{ ansible_distribution_release }}_backports_preferences'
+    dest: /etc/apt/preferences.d/0-backports-defaults
+    force: yes
+    mode: "0640"
+  register: apt_backports_config
+  tags:
+    - apt
+
+- name: Apt update
+  apt:
+    update_cache: yes
+  when: apt_backports_sources is changed or apt_backports_config is changed
+  tags:
+    - apt

apt/tasks/backports.oneline.yml

@@ -33,13 +33,13 @@
     line: 'Acquire::Check-Valid-Until no;'
     create: yes
     state: present
-  when: ansible_distribution_release == "jessie"
   tags:
     - apt
+  when: ansible_distribution_release == "jessie"
 
 - name: Apt update
   apt:
     update_cache: yes
-  when: apt_backports_list is changed or apt_backports_config is changed
   tags:
     - apt
+  when: apt_backports_list is changed or apt_backports_config is changed

apt/tasks/basics.deb822.yml

@@ -0,0 +1,28 @@
+---
+
+- name: Change basics repositories
+  template:
+    src: "{{ ansible_distribution_release }}_basics.sources.j2"
+    dest: /etc/apt/sources.list.d/system.sources
+    mode: "0644"
+    force: yes
+  register: apt_basic_sources
+  tags:
+    - apt
+
+- name: Change security repositories
+  template:
+    src: "{{ ansible_distribution_release }}_security.sources.j2"
+    dest: /etc/apt/sources.list.d/security.sources
+    mode: "0644"
+    force: yes
+  register: apt_security_sources
+  tags:
+    - apt
+
+- name: Apt update
+  apt:
+    update_cache: yes
+  tags:
+    - apt
+  when: apt_basic_list is changed or apt_security_sources is changed

apt/tasks/basics.oneline.yml

@@ -0,0 +1,18 @@
+---
+
+- name: Change basics repositories
+  template:
+    src: "{{ ansible_distribution_release }}_basics.list.j2"
+    dest: /etc/apt/sources.list
+    mode: "0644"
+    force: yes
+  register: apt_basic_list
+  tags:
+    - apt
+
+- name: Apt update
+  apt:
+    update_cache: yes
+  tags:
+    - apt
+  when: apt_basic_list is changed

apt/tasks/basics.yml

@@ -1,33 +0,0 @@
----
-
-- name: Change basics repositories
-  template:
-    src: "{{ ansible_distribution_release }}_basics.list.j2"
-    dest: /etc/apt/sources.list
-    mode: "0644"
-    force: yes
-  register: apt_basic_list
-  tags:
-    - apt
-
-- name: Clean GANDI sources.list.d/debian-security.list
-  file:
-    path: '{{ item }}'
-    state: absent
-  loop:
-    - /etc/apt/sources.list.d/debian-security.list
-    - /etc/apt/sources.list.d/debian-jessie.list
-    - /etc/apt/sources.list.d/debian-stretch.list
-    - /etc/apt/sources.list.d/debian-buster.list
-    - /etc/apt/sources.list.d/debian-bullseye.list
-    - /etc/apt/sources.list.d/debian-update.list
-  when: apt_clean_gandi_sourceslist | bool
-  tags:
-    - apt
-
-- name: Apt update
-  apt:
-    update_cache: yes
-  when: apt_basic_list is changed
-  tags:
-    - apt

apt/tasks/evolix_public.deb822.yml

@@ -0,0 +1,45 @@
+---
+
+- name: Look for legacy apt keyring
+  stat:
+    path: /etc/apt/trusted.gpg
+  register: _trusted_gpg_keyring
+  tags:
+    - apt
+
+- name: Evolix embedded GPG key is absent
+  apt_key:
+    id: "B8612B5D"
+    keyring: /etc/apt/trusted.gpg
+    state: absent
+  tags:
+    - apt
+  when: _trusted_gpg_keyring.stat.exists
+
+- name: Add Evolix GPG key
+  copy:
+    src: pub_evolix.asc
+    dest: "{{ apt_keyring_dir }}/pub_evolix.asc"
+    force: yes
+    mode: "0644"
+    owner: root
+    group: root
+  tags:
+    - apt
+
+- name: Evolix public list is installed
+  template:
+    src: evolix_public.sources.j2
+    dest: /etc/apt/sources.list.d/evolix_public.sources
+    force: yes
+    mode: "0640"
+  register: apt_evolix_public
+  tags:
+    - apt
+
+- name: Apt update
+  apt:
+    update_cache: yes
+  tags:
+    - apt
+  when: apt_evolix_public is changed

apt/tasks/evolix_public.oneline.yml

@@ -12,9 +12,9 @@
     id: "B8612B5D"
     keyring: /etc/apt/trusted.gpg
     state: absent
-  when: _trusted_gpg_keyring.stat.exists
   tags:
     - apt
+  when: _trusted_gpg_keyring.stat.exists
 
 - name: Add Evolix GPG key
   copy:
@@ -40,6 +40,6 @@
 - name: Apt update
   apt:
     update_cache: yes
-  when: apt_evolix_public is changed
   tags:
     - apt
+  when: apt_evolix_public is changed

apt/tasks/hold_packages.yml

@@ -97,6 +97,6 @@
     day: "{{ apt_check_hold_cron_day }}"
     month: "{{ apt_check_hold_cron_month }}"
     state: "present"
-  when: is_cron.rc == 0
   tags:
     - apt
+  when: is_cron.rc == 0

apt/tasks/main.yml

@@ -1,10 +1,26 @@
 ---
 
 - name: "Compatibility check"
-  fail:
-    msg: only compatible with Debian >= 8
-  when:
-    - ansible_distribution != "Debian" or ansible_distribution_major_version is version('8', '<')
+  assert:
+    that:
+      - ansible_distribution = "Debian"
+      - ansible_distribution_major_version is version('8', '>=')
+    msg: Only compatible with Debian >= 8
+  tags:
+    - apt
+
+- name: "apt-transport-https is installed for https repositories (before Buster)"
+  apt:
+    name:
+      - apt-transport-https
+  tags:
+    - apt
+  when: ansible_distribution_major_version is version('10', '<')
+
+- name: "certificates are installed to https repositories"
+  apt:
+    name:
+      - ca-certificates
   tags:
     - apt
 
@@ -14,23 +30,71 @@
   tags:
     - apt
 
-- name: Install basics repositories
-  include: basics.yml
-  when: apt_install_basics | bool
+- name: Install basics repositories (Debian <12)
+  include: basics.debian-lt-12.yml
   tags:
     - apt
+  when:
+    - apt_install_basics | bool
+    - ansible_distribution_major_version is version('12', '<')
 
-- name: Install APT Backports repository
-  include: backports.yml
-  when: apt_install_backports | bool
+- name: Install basics repositories (Debian >=12)
+  include: basics.debian-ge-12.yml
   tags:
     - apt
+  when:
+    - apt_install_basics | bool
+    - ansible_distribution_major_version is version('12', '>=')
 
-- name: Install Evolix Public APT repository
-  include: evolix_public.yml
-  when: apt_install_evolix_public | bool
+
+- name: Install backports repositories (Debian <12)
+  include: backports.debian-lt-12.yml
   tags:
     - apt
+  when:
+    - apt_install_backports | bool
+    - ansible_distribution_major_version is version('12', '<')
+
+- name: Install backports repositories (Debian >=12)
+  include: backports.debian-ge-12.yml
+  tags:
+    - apt
+  when:
+    - apt_install_backports | bool
+    - ansible_distribution_major_version is version('12', '>=')
+
+
+- name: Install Evolix Public repositories (Debian <12)
+  include: evolix_public.debian-lt-12.yml
+  tags:
+    - apt
+  when:
+    - apt_install_evolix_public | bool
+    - ansible_distribution_major_version is version('12', '<')
+
+- name: Install Evolix Public repositories (Debian >=12)
+  include: evolix_public.debian-ge-12.yml
+  tags:
+    - apt
+  when:
+    - apt_install_evolix_public | bool
+    - ansible_distribution_major_version is version('12', '>=')
+
+- name: Clean GANDI sources
+  file:
+    path: '{{ item }}'
+    state: absent
+  loop:
+    - /etc/apt/sources.list.d/debian-security.list
+    - /etc/apt/sources.list.d/debian-jessie.list
+    - /etc/apt/sources.list.d/debian-stretch.list
+    - /etc/apt/sources.list.d/debian-buster.list
+    - /etc/apt/sources.list.d/debian-bullseye.list
+    - /etc/apt/sources.list.d/debian-update.list
+  tags:
+    - apt
+  when: apt_clean_gandi_sourceslist | bool
+
 
 - name: Install check for packages marked hold
   include: hold_packages.yml

apt/templates/bookworm_backports.sources.j2

@@ -0,0 +1,7 @@
+# {{ ansible_managed }}
+
+Types: deb
+URIs: https://mirror.evolix.org/debian
+Suites: bullseye-backports
+Components: {{ apt_backports_components | mandatory }}
+Enabled: yes

apt/templates/bookworm_basics.list.j2

@@ -1,5 +0,0 @@
-# {{ ansible_managed }}
-
-deb http://mirror.evolix.org/debian bookworm {{ apt_basics_components | mandatory }}
-deb http://mirror.evolix.org/debian/ bookworm-updates {{ apt_basics_components | mandatory }} 
-deb http://security.debian.org/debian-security bookworm-security {{ apt_basics_components | mandatory }}

apt/templates/bookworm_basics.sources.j2

@@ -0,0 +1,9 @@
+# {{ ansible_managed }}
+
+Types: deb
+URIs: https://mirror.evolix.org/debian
+Suites: bookworm bookworm-updates
+Components: {{ apt_basics_components | mandatory }} 
+Enabled: yes
+
+deb http://security.debian.org/debian-security bookworm-security {{ apt_basics_components | mandatory }}

apt/templates/bookworm_security.sources.j2

@@ -0,0 +1,7 @@
+# {{ ansible_managed }}
+
+Types: deb
+URIs: https://security.debian.org/debian-security
+Suites: bookworm-security
+Components: {{ apt_basics_components | mandatory }} 
+Enabled: yes