Improve downloaded deployment image support

This change introduces various improvements surrounding deployment
images that are downloaded rather than built locally.

Default images for Centos 8/9 Stream, Ubuntu 20.04/22.04, and Rocky
Linux 8/9 have all been added. These can be selected by setting
upstream_deploy_image_distribution and upstream_deploy_image_release,
and are deployed when download_custom_deploy_image is true.

The deployment image will not be re-downloaded if it already exists, and
its checksum matches

Change-Id: Ib0a1dac2bd0c4c3cf721bcf3042dbd34e2ab2dbd

playbooks/roles/bifrost-ironic-install/defaults/main.yml

@@ -114,11 +114,59 @@ update_repos: true
 update_ipa: "{{ update_repos }}"
 # Use the DIB dynamic-login element to insert the SSH key
 ipa_add_ssh_key: false
-# NOTE(Alex-Welsh): cirros_deploy_image_upstream_url has been deprecated in
-# favor of custom_deploy_image_upstream_url but is included for backwards
-# compatibility. It should be swapped permanently to
-# custom_deploy_image_upstream_url in the next major release.
-custom_deploy_image_upstream_url: "{{ cirros_deploy_image_upstream_url | default('https://download.cirros-cloud.net/0.5.3/cirros-0.5.3-x86_64-disk.img') }}"
+
+# Deployment image distribution, for selecting a default upstream image. Valid
+# options are "cirros", "centos", "rocky", "ubuntu". Default is "cirros".
+upstream_deploy_image_distribution: "cirros"
+# Deployment image releases and default values.
+default_upstream_deploy_image_releases:
+  cirros: "0.5.3"
+  centos: "9-stream"
+  rocky: "9"
+  ubuntu: "jammy"
+# Deployment image release, for selecting a default upstream image.
+upstream_deploy_image_release: "{{ default_upstream_deploy_image_releases[upstream_deploy_image_distribution] }}"
+deploy_image_sources:
+  ubuntu:
+    focal:
+      image: "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img"
+      checksum: "https://cloud-images.ubuntu.com/focal/current/SHA256SUMS"
+      checksum_algorithm: "sha256"
+    jammy:
+      image: "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img"
+      checksum: "https://cloud-images.ubuntu.com/jammy/current/SHA256SUMS"
+      checksum_algorithm: "sha256"
+  rocky:
+    "8":
+      image: "https://dl.rockylinux.org/pub/rocky/8/images/x86_64/Rocky-8-GenericCloud.latest.x86_64.qcow2"
+      checksum: "https://dl.rockylinux.org/pub/rocky/8/images/x86_64/CHECKSUM"
+      checksum_algorithm: "sha256"
+    "9":
+      image: "https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2"
+      checksum: "https://dl.rockylinux.org/pub/rocky/9/images/x86_64/CHECKSUM"
+      checksum_algorithm: "sha256"
+  centos:
+    8-stream:
+      image: "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20230501.0.x86_64.qcow2"
+      checksum: "https://cloud.centos.org/centos/8-stream/x86_64/images/CHECKSUM"
+      checksum_algorithm: "sha256"
+    9-stream:
+      image: "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-20230501.0.x86_64.qcow2"
+      checksum: "https://cloud.centos.org/centos/9-stream/x86_64/images/CHECKSUM"
+      checksum_algorithm: "sha256"
+  cirros:
+    "0.5.3":
+      # NOTE(Alex-Welsh): cirros_deploy_image_upstream_url has been deprecated in
+      # favor of custom_deploy_image_upstream_url but is included for backwards
+      # compatibility. It should be swapped permanently to
+      # custom_deploy_image_upstream_url in the next major release.
+      image: "{{ cirros_deploy_image_upstream_url | default('https://download.cirros-cloud.net/0.5.3/cirros-0.5.3-x86_64-disk.img') }}"
+      checksum: "https://download.cirros-cloud.net/0.5.3/MD5SUMS"
+      checksum_algorithm: "md5"
+deploy_image_upstream_url: "{{ custom_deploy_image_upstream_url | default(deploy_image_sources[upstream_deploy_image_distribution][upstream_deploy_image_release].image) }}"
+deploy_image_checksum_url: "{{ custom_deploy_image_checksum_url | default(deploy_image_sources[upstream_deploy_image_distribution][upstream_deploy_image_release].checksum) }}"
+deploy_image_checksum_algorithm: "{{ custom_deploy_image_checksum_algorithm | default(deploy_image_sources[upstream_deploy_image_distribution][upstream_deploy_image_release].checksum_algorithm) }}"
+
 # By default bifrost will deploy dnsmasq to utilize as an integrated DHCP
 # server. If you already have a DHCP server or do not need DHCP/TFTP at all,
 # you can pass --disable-dhcp to the bifrost-cli install command to disable it.

playbooks/roles/bifrost-ironic-install/tasks/bootstrap.yml

@@ -381,37 +381,9 @@
     - not create_ipa_image | bool
     - download_ipa | bool
 
-- when: download_custom_deploy_image | bool
-  block:
-    - name: "Download a custom image to use for deployment if requested"
-      get_url:
-        url: "{{ custom_deploy_image_upstream_url }}"
-        dest: "{{ deploy_image }}"
-        owner: ironic
-        group: ironic
-        mode: "0644"
-  rescue:
-    - name: "Attempt to use curl if get_url fails"
-      command: "curl -L --output {{ deploy_image }} {{ custom_deploy_image_upstream_url }}" # noqa: command-instead-of-module
-    - name: Set file permissions
-      file:
-        path: "{{ deploy_image }}"
-        owner: ironic
-        group: ironic
-        mode: "0644"
-
-- when: download_custom_deploy_image | bool
-  block:
-    - name: "Create a checksum file for the custom deployment image"
-      shell: sha256sum {{ deploy_image_filename }} > {{ deploy_image_filename }}.CHECKSUMS
-      args:
-        chdir: "{{ http_boot_folder }}"
-    - name: "Ensure the checksum file is readable"
-      file:
-        path: "{{ http_boot_folder }}/{{ deploy_image_filename }}.CHECKSUMS"
-        owner: ironic
-        group: ironic
-        mode: "0644"
+- name: "Download deployment image"
+  include_tasks: download_deployment_image.yml
+  when: download_custom_deploy_image | bool
 
 - name: "Bootstrap Nginx"
   import_role:

playbooks/roles/bifrost-ironic-install/tasks/download_deployment_image.yml

@@ -0,0 +1,72 @@
+---
+- name: "Gather details of existing deployment image"
+  stat:
+    path: "{{ http_boot_folder }}/{{ deploy_image_filename }}"
+    checksum_algorithm: "{{ omit if deploy_image_checksum_algorithm == 'none' else deploy_image_checksum_algorithm }}"
+    get_checksum: "{{ deploy_image_checksum_algorithm != 'none' }}"
+  register: deploy_image_stat
+
+- block:
+    - name: "Download checksum file for custom deployment image"
+      get_url:
+        url: "{{ deploy_image_checksum_url }}"
+        dest: "{{ http_boot_folder }}/{{ deploy_image_filename }}-checksum.CHECKSUMS"
+        force: true
+        owner: ironic
+        group: ironic
+        mode: "0644"
+  rescue:
+    - name: "Attempt to use curl if get_url fails"
+      command: "curl -L --output {{ http_boot_folder }}/{{ deploy_image_filename }}-checksum.CHECKSUMS {{ deploy_image_checksum_url }}" # noqa: command-instead-of-module
+      no_log: true
+    - name: Set checksum file permissions
+      file:
+        path: "{{ http_boot_folder }}/{{ deploy_image_filename }}-checksum.CHECKSUMS"
+        owner: ironic
+        group: ironic
+        mode: "0644"
+
+- name: "Extract deployment image checksum"
+  command: awk '/{{ deploy_image_upstream_url | basename }}|^[a-z0-9]+$/{print $1}' {{ http_boot_folder }}/{{ deploy_image_filename }}-checksum.CHECKSUMS
+  register: parsed_deployment_image_checksum
+  failed_when: parsed_deployment_image_checksum is failed
+               or not parsed_deployment_image_checksum.stdout
+
+- set_fact:
+    deployment_image_checksum: "{{ deploy_image_checksum_algorithm }}:{{ parsed_deployment_image_checksum.stdout }}"
+
+- when: >-
+    (deploy_image_stat.stat.checksum | default("none") not in parsed_deployment_image_checksum.stdout | default("")) or
+    (not deploy_image_stat.stat.exists)
+  block:
+    - name: "Download a deployment image"
+      get_url:
+        url: "{{ deploy_image_upstream_url }}"
+        dest: "{{ deploy_image }}"
+        owner: ironic
+        group: ironic
+        mode: "0644"
+        checksum: "{{ deployment_image_checksum }}"
+  rescue:
+    - name: "Attempt to use curl if get_url fails"
+      command: "curl -L --output {{ deploy_image }} {{ deploy_image_upstream_url }}" # noqa: command-instead-of-module
+    - name: Set file permissions
+      file:
+        path: "{{ deploy_image }}"
+        owner: ironic
+        group: ironic
+        mode: "0644"
+    - name: "Gather details of new deployment image"
+      stat:
+        path: "{{ deploy_image }}"
+        checksum_algorithm: "{{ deploy_image_checksum_algorithm }}"
+        get_checksum: true
+      register: downloaded_image_stat
+      when: deploy_image_checksum_algorithm != 'none'
+    - name: Fail when downloaded image does not match checksum
+      fail:
+        msg: |
+          The deployment image checksum does not match the file that has been downloaded.
+          Ensure that deploy_image_upstream_url, deploy_image_checksum_url, and deploy_image_checksum_algorithm are set correctly.
+          To ignore the checksum, set deploy_image_checksum_algorithm to none.
+      when: deploy_image_checksum_algorithm != 'none' and downloaded_image_stat.stat.checksum != parsed_deployment_image_checksum

releasenotes/notes/downloaded-deploy-image-improvements-2563ffb57a41ab20.yaml

@@ -0,0 +1,13 @@
+---
+features:
+  - |
+    Default images for Centos 8/9 Stream, Ubuntu 20.04/22.04, and Rocky Linux
+    8/9 have been added. These can be selected by setting
+    ``upstream_deploy_image_distribution`` and
+    ``upstream_deploy_image_release``, and are deployed when
+    ``download_custom_deploy_image`` is true. The default image remains CirrOS.
+  - |
+    When ``download_custom_deploy_image`` is true, the image will no longer be
+    re-downloaded if the image checksum matches upstream. For custom images,
+    the checksum can be changed using ``custom_deploy_image_checksum_url`` and
+    ``custom_deploy_image_checksum_algorithm``.