From 9e97d5646a68d4f0df2970327dbe9e69277ecc62 Mon Sep 17 00:00:00 2001
From: mciecier <mciecier@redhat.com>
Date: Thu, 2 Feb 2023 18:25:20 +0100
Subject: [PATCH] Check podman containers update during logs collection

Add validation for podman containers update

Change-Id: I4329e250d811628dbc2c1cf906052458f1bcc649
---
 defaults/main.yml              |  5 +++++
 templates/collect_logs.sh.j2   |  3 ++-
 templates/collect_logs.yaml.j2 | 39 +++++++++++++++++++++++++++++++++-
 3 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/defaults/main.yml b/defaults/main.yml
index 953b0f8e..d9efc2fd 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -158,6 +158,11 @@ containers_default_parameters: "{{ working_dir }}/docker-osp12.yaml"
 # container_images.yaml file location
 container_images_location: "{{ working_dir }}/container_images.yaml"
 
+# comma seperated list of container names that are expected to be not updated before container test
+# Exclude nova_virtlogd by default, because container iamge test happens before overcloud reboot
+# https://opendev.org/openstack/tripleo-heat-templates/src/commit/64a52f31507f464a0437aac0a53f65250845324b/releasenotes/notes/nova_virtlogd_wrapper-120fcfcfa0787b2b.yaml
+excluded_containers_from_check: "nova_virtlogd"
+
 # undercloud_hiera.yaml file location
 undercloud_hiera: "{{ working_dir }}/undercloud_hiera.yaml"
 
diff --git a/templates/collect_logs.sh.j2 b/templates/collect_logs.sh.j2
index 0be96327..b6ae9318 100644
--- a/templates/collect_logs.sh.j2
+++ b/templates/collect_logs.sh.j2
@@ -4,6 +4,7 @@
 
 CURRENT_STAGE=${1:-{{ log_current_stage }}}
 SSH_USER={{ (overcloud_ssh_user) | ternary(overcloud_ssh_user, 'tripleo-admin') }}
+EXCLUDED_CONTAINERS_FROM_CHECK=${3:-{{ excluded_containers_from_check }}}
 
 # This should always be true for tripleo>=wallaby.
 if [ ! -f {{ upgrade_validation_inventory }} ]; then
@@ -21,4 +22,4 @@ else
     INVENTORY={{ upgrade_validation_inventory }}
 fi
 
-ansible-playbook -i "${INVENTORY}" -e current_stage="${CURRENT_STAGE}" {{ log_playbook }}
+ansible-playbook -i "${INVENTORY}" -e current_stage="${CURRENT_STAGE}" -e containers_check_excluded="${EXCLUDED_CONTAINERS_FROM_CHECK}" {{ log_playbook }}
diff --git a/templates/collect_logs.yaml.j2 b/templates/collect_logs.yaml.j2
index d5dacd12..6ae590f6 100644
--- a/templates/collect_logs.yaml.j2
+++ b/templates/collect_logs.yaml.j2
@@ -19,8 +19,11 @@
         dnf list installed &>> /var/log/extra/packages-{% raw %}{{ current_stage }}{% endraw %}.txt
 
     - name: get podman container state at this stage
+      vars:
+        f_open: '{% raw %}{{"{{"}}{% endraw %}'
+        f_close: '{% raw %}{{"}}"}}{% endraw %}'
       shell: |
-        podman ps --all &>> /var/log/extra/container-ps-{% raw %}{{ current_stage }}{% endraw %}.txt
+        podman ps  --format {% raw %}"{{ f_open }}.Names{{ f_close }} {{ f_open }}.ID{{ f_close }} {{ f_open }}.Image{{ f_close }}"{% endraw %} | sort &>> /var/log/extra/container-ps-{% raw %}{{ current_stage }}{% endraw %}.txt
 
     - name: get podman images state at this stage
       shell: |
@@ -50,6 +53,40 @@
           fi
         done
 
+    - name: get container files and test container images update
+      block:
+        - name: get all logs files related to container
+          find:
+            paths: "/var/log/extra/"
+            patterns: 'container-ps-before_*.txt'
+          register: found_files
+
+        - name: get the oldest file related to container states
+          set_fact:
+            latest_log: "{% raw %}{{ found_files.files | sort(attribute='mtime') | first }}{% endraw %}"
+
+        - name: create grep string for containers excluded from test
+          set_fact:
+            grep_exclude: "{% raw %}| grep -v -e {{ containers_check_excluded|split(',')|join(' -e ') }}{% endraw %}"
+          when: containers_check_excluded != ""
+
+        - name: check if podman images are updated before reboot
+          shell: "comm --nocheck-order -12 /var/log/extra/container-ps-before_reboot.txt {% raw %}{{ latest_log.path }} {{ grep_exclude|default('') }}{% endraw %}"
+          register: comm_result
+          failed_when: comm_result.rc >= 2
+
+        - name: not all podman images updated
+          fail:
+            msg: "Not all images updated"
+          when: comm_result.stdout != ""
+      when:
+        - current_stage == "before_reboot"
+        - not {{ tripleo_ci|bool }}
+      always:
+        - name: container images comparison result
+          debug:
+            var: comm_result.stdout
+
 - hosts: ovn_controller
   gather_facts: false
   become: true