[UPDATES] Validated oc containers/images.

This change ensures that docker containers after minor update
 use docker images from file passed with --container-registry
option.

Related-Bug: 1738142

Change-Id: I926b6526d6bdc707cb88965a93709bf8514504ae

defaults/main.yml

@@ -69,6 +69,10 @@ post_overcloud_update_workarounds_script: "{{ working_dir }}/post_overcloud_upda
 overcloud_update_setup_script: "{{ working_dir }}/overcloud_update_setup.sh"
 overcloud_update_script_base: "{{ working_dir }}/overcloud_update"
 
+overcloud_validate_images_template: validate_docker_images_versions.sh.j2
+overcloud_validate_images_script: "{{ working_dir }}/validate_docker_images_versions.sh"
+
+
 # container registry file
 container_registry_file: "docker-images.yaml"
 

tasks/update/create-update-scripts.yaml

@@ -45,3 +45,10 @@
     force: true
   with_items:
     - "{{ oc_roles.stdout_lines|default('all') }}"
+
+- name: create docker container/images validate scripts
+  template:
+    src: "{{ overcloud_validate_images_template }}"
+    dest: "{{ overcloud_validate_images_script }}"
+    mode: 0755
+    force: true

tasks/update/main.yml

@@ -65,6 +65,13 @@
           bash {{ post_overcloud_update_workarounds_script }} &> {{ working_dir }}/post_overcloud_update_workarounds.log
       when: updates_workarounds|bool
 
+    - name: validate overcloud docker images/containers
+      shell: |
+          source {{ undercloud_rc }}
+          bash {{ overcloud_validate_images_script }} &> {{ working_dir}}/validate_oc_images_containers.log
+      tags:
+          - overcloud_update
+
     - name: stop l3 agent connectivity check
       shell: |
         source {{ overcloud_rc }}

templates/validate_docker_images_versions.sh.j2

@@ -0,0 +1,71 @@
+#!/bin/env bash
+set -euo pipefail
+
+function assert_image_in_file {
+    #
+    # Assert image used by running container is present in the registry
+    # Arg 1 - container-registry file
+    # Arg 2 - image to check
+    #
+
+    local _image_file=${1}
+    local _image_name=${2}
+
+    if [[ -z ${_image_file} || -z ${_image_name} ]] ; then
+        echo "ERROR: Fuction assert_image_in_file requires 2 arguments"
+        exit 2
+    fi
+
+    set +e
+    grep -q "${_image_name}" "${_image_file}"
+
+    if [[ ${?} -ne 0 ]] ; then
+        echo "Image ${_image_name} not present in ${_image_file}"
+        exit 2
+    else
+        set -e
+        echo "Image ${_image_name} present in ${_image_file}"
+    fi
+
+}
+
+source {{ undercloud_rc }}
+for _ip in $( openstack server list -f value -c Networks | awk -F '=' '{ print $2 }' ) ; do
+    echo "================================================================================"
+    echo "Validate docker images at host ${_ip}"
+    echo "================================================================================"
+    CONTAINERS=$( ssh -q -o StrictHostKeyChecking=no heat-admin@${_ip} \
+                    "sudo docker ps 2>&1 | grep -v -e pcmk -e ID | awk '{ print \$2 }'" )
+    for _container in ${CONTAINERS} ; do
+        # non pcs managed containers
+        assert_image_in_file {{ working_dir}}/{{ container_registry_file }} ${_container}
+    done
+
+    CONTAINERS_PCMK=$( ssh -q -o StrictHostKeyChecking=no heat-admin@${_ip} \
+                        "sudo docker images 2>&1 | grep pcmk | awk '{ print \$3 }'" )
+
+    if [[ -z ${CONTAINERS_PCMK} ]] ; then
+        echo "PCMKLATEST images not found on ${_ip}"
+    else
+        echo "================================================================================"
+        echo "Validate PCMKLATEST docker images at host ${_ip}"
+        echo "================================================================================"
+        for _img_id in ${CONTAINERS_PCMK} ; do
+            _pcmk_image=$( ssh -q -o StrictHostKeyChecking=no heat-admin@${_ip} \
+                            "sudo docker images 2>&1 | grep ${_img_id} | grep -v pcmklatest | awk '{ print \$1\":\"\$2}'" )
+            assert_image_in_file {{ working_dir }}/{{ container_registry_file }} ${_pcmk_image}
+
+            _pcmk_container=$( ssh -q -o StrictHostKeyChecking=no heat-admin@${_ip} \
+                                "sudo docker ps --filter ancestor=${_img_id} | grep pcmklatest | awk '{print \$2}'" )
+            if [[ -z ${_pcmk_container} ]] ; then
+                echo "PCMKLATEST container for image '${_pcmk_image}' is not running"
+                echo "Running container is '${_pcmk_container}'"
+                exit 2
+            else
+                echo "PCMKLATES container ${_pcmk_container} usese image ${_pcmk_image}"
+            fi
+
+        done
+    fi
+
+done