Check podman containers update during logs collection

Add validation for podman containers update

Change-Id: I4329e250d811628dbc2c1cf906052458f1bcc649

defaults/main.yml

@@ -158,6 +158,11 @@ containers_default_parameters: "{{ working_dir }}/docker-osp12.yaml"
 # container_images.yaml file location
 container_images_location: "{{ working_dir }}/container_images.yaml"
 
+# comma seperated list of container names that are expected to be not updated before container test
+# Exclude nova_virtlogd by default, because container iamge test happens before overcloud reboot
+# https://opendev.org/openstack/tripleo-heat-templates/src/commit/64a52f31507f464a0437aac0a53f65250845324b/releasenotes/notes/nova_virtlogd_wrapper-120fcfcfa0787b2b.yaml
+excluded_containers_from_check: "nova_virtlogd"
+
 # undercloud_hiera.yaml file location
 undercloud_hiera: "{{ working_dir }}/undercloud_hiera.yaml"
 

templates/collect_logs.sh.j2

@@ -4,6 +4,7 @@
 
 CURRENT_STAGE=${1:-{{ log_current_stage }}}
 SSH_USER={{ (overcloud_ssh_user) | ternary(overcloud_ssh_user, 'tripleo-admin') }}
+EXCLUDED_CONTAINERS_FROM_CHECK=${3:-{{ excluded_containers_from_check }}}
 
 # This should always be true for tripleo>=wallaby.
 if [ ! -f {{ upgrade_validation_inventory }} ]; then
@@ -21,4 +22,4 @@ else
     INVENTORY={{ upgrade_validation_inventory }}
 fi
 
-ansible-playbook -i "${INVENTORY}" -e current_stage="${CURRENT_STAGE}" {{ log_playbook }}
+ansible-playbook -i "${INVENTORY}" -e current_stage="${CURRENT_STAGE}" -e containers_check_excluded="${EXCLUDED_CONTAINERS_FROM_CHECK}" {{ log_playbook }}

templates/collect_logs.yaml.j2

@@ -19,8 +19,11 @@
         dnf list installed &>> /var/log/extra/packages-{% raw %}{{ current_stage }}{% endraw %}.txt
 
     - name: get podman container state at this stage
+      vars:
+        f_open: '{% raw %}{{"{{"}}{% endraw %}'
+        f_close: '{% raw %}{{"}}"}}{% endraw %}'
       shell: |
-        podman ps --all &>> /var/log/extra/container-ps-{% raw %}{{ current_stage }}{% endraw %}.txt
+        podman ps  --format {% raw %}"{{ f_open }}.Names{{ f_close }} {{ f_open }}.ID{{ f_close }} {{ f_open }}.Image{{ f_close }}"{% endraw %} | sort &>> /var/log/extra/container-ps-{% raw %}{{ current_stage }}{% endraw %}.txt
 
     - name: get podman images state at this stage
       shell: |
@@ -50,6 +53,40 @@
           fi
         done
 
+    - name: get container files and test container images update
+      block:
+        - name: get all logs files related to container
+          find:
+            paths: "/var/log/extra/"
+            patterns: 'container-ps-before_*.txt'
+          register: found_files
+
+        - name: get the oldest file related to container states
+          set_fact:
+            latest_log: "{% raw %}{{ found_files.files | sort(attribute='mtime') | first }}{% endraw %}"
+
+        - name: create grep string for containers excluded from test
+          set_fact:
+            grep_exclude: "{% raw %}| grep -v -e {{ containers_check_excluded|split(',')|join(' -e ') }}{% endraw %}"
+          when: containers_check_excluded != ""
+
+        - name: check if podman images are updated before reboot
+          shell: "comm --nocheck-order -12 /var/log/extra/container-ps-before_reboot.txt {% raw %}{{ latest_log.path }} {{ grep_exclude|default('') }}{% endraw %}"
+          register: comm_result
+          failed_when: comm_result.rc >= 2
+
+        - name: not all podman images updated
+          fail:
+            msg: "Not all images updated"
+          when: comm_result.stdout != ""
+      when:
+        - current_stage == "before_reboot"
+        - not {{ tripleo_ci|bool }}
+      always:
+        - name: container images comparison result
+          debug:
+            var: comm_result.stdout
+
 - hosts: ovn_controller
   gather_facts: false
   become: true