From 3efa98a31b9e3e52d5c1add680c8ccc65abc48f1 Mon Sep 17 00:00:00 2001 From: Roman Gorshunov Date: Wed, 5 Sep 2018 19:26:49 +0200 Subject: [PATCH] versions.yaml updater tool: verify that images & tags do exist This patch allows the tool to verify that container images with specific tags equal to the HEAD git commit id's we reference, do really exist on quay.io repository and are available for download. Change-Id: I2205f4fa63a085d94ef3c0e9d2ff511665df43e6 --- tools/updater.py | 108 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 4 deletions(-) diff --git a/tools/updater.py b/tools/updater.py index 946edbc0a..a7e37e035 100755 --- a/tools/updater.py +++ b/tools/updater.py @@ -23,12 +23,18 @@ # 2) tags of container images listed in dict `image_repo_git_url` in the # code below # +# In addition to that, the tool verifies that container images with +# specific tags equal to the HEAD git commit id's we reference, do really +# exist on quay.io repository and are available for download. +# from functools import reduce import argparse +import datetime import logging import operator import os +import requests import sys try: @@ -54,7 +60,8 @@ image_repo_git_url = { 'quay.io/airshipit/airflow': 'https://git.openstack.org/openstack/airship-shipyard', 'quay.io/airshipit/armada': 'https://git.openstack.org/openstack/airship-armada', 'quay.io/airshipit/deckhand': 'https://git.openstack.org/openstack/airship-deckhand', - 'quay.io/airshipit/divingbell': 'https://git.openstack.org/openstack/airship-divingbell', + # yes, divingbell image is just Ubuntu 16.04 image, and we don't check it's tag + #'docker.io/ubuntu': 'https://git.openstack.org/openstack/airship-divingbell', 'quay.io/airshipit/drydock': 'https://git.openstack.org/openstack/airship-drydock', # maas-{rack,region}-controller images are built from airship-maas repository 'quay.io/airshipit/maas-rack-controller': 'https://git.openstack.org/openstack/airship-maas', @@ -64,18 +71,33 @@ image_repo_git_url = { 'quay.io/airshipit/shipyard': 'https://git.openstack.org/openstack/airship-shipyard', # sstream-cache image is built from airship-maas repository 'quay.io/airshipit/sstream-cache': 'https://git.openstack.org/openstack/airship-maas', - 'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios' - # Disabled by Kaspars: https://review.openstack.org/#/c/596909/21/tools/updater.py@53 - #'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter' + 'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios', + 'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter' } logging.basicConfig(level=logging.INFO) # Temporary dict of git url's and cached commit id's: {'git_url': 'commit_id'} +global git_url_commit_ids git_url_commit_ids = {} +# Temporary dict of image repo's and status of image on quay.io +global image_repo_status +image_repo_status = {} dict_path = None +def inverse_dict(dic): + """Accepts dictionary, returns dictionary where keys become values, + and values become keys""" + new_dict = {} + for k, v in dic.items(): + new_dict[v] = k + return new_dict + + +git_url_image_repo = inverse_dict(image_repo_git_url) + + # https://stackoverflow.com/a/35585837 def lsremote(url, remote_ref): """Accepts git url and remote reference, returns git commit id.""" @@ -92,14 +114,92 @@ def get_commit_id(url): # If we don't have this git url in our url's dictionary, # fetch latest commit ID and add new dictionary entry logging.debug('git_url_commit_ids: %s', git_url_commit_ids) + logging.debug('image_repo_status: %s', image_repo_status) if url not in git_url_commit_ids: logging.debug('git url: ' + url + ' is not in git_url_commit_ids dict;' + ' adding it with HEAD commit id') git_url_commit_ids[url] = lsremote(url, 'HEAD') + if url in git_url_image_repo: + if git_url_image_repo[url] not in image_repo_status: + image_repo_status[git_url_image_repo[url]] = \ + verify_image_tag(git_url_image_repo[url], git_url_commit_ids[url]) + else: + logging.debug('We checked image ' + git_url_image_repo[url] + + ' on quay.io already, skipping') return git_url_commit_ids[url] +def verify_image_tag(image, git_commit_id): + """Verify if image with certain tag exists on quay.io, + returns 0 (image not hosted on quay.io), True, or False + """ + if not image.startswith('quay.io/'): + logging.info('Unable to verify if image ' + image + ':' + + git_commit_id + ' with this specific tag exists' + + ' in containers repository: only quay.io is' + + ' supported at the moment') + return 0 + + logging.info('Verifying if image ' + image + ':' + git_commit_id + + ' with this specific tag exists on quay.io...') + + retries = 0 + max_retries = 5 + + payload = {'specificTag': git_commit_id} + hash_image = image.split('/') + url = 'https://quay.io/api/v1/repository/' + \ + hash_image[1] + '/' + hash_image[2] + '/tag/' + + while retries <= max_retries: + retries = retries +1 + try: + res = requests.get(url, params = payload, timeout = 5) + except requests.exceptions.Timeout: + logging.warning("Failed to fetch url %s" % res.url) + sleep(1) + if retries == max_retries: + logging.error("Failed to connect to quay.io") + return 0 + + if res.status_code != 200: + logging.error('Image %s is not available on quay.io or it' + + ' requires authentication', image) + return 0 + + try: + res = res.json() + except json.decoder.JSONDecodeError: + logging.error('Unable to parse response from quay.io (%s)' % res.url) + return 0 + + try: + for tag in res['tags']: + # Normally there should be only one tag description + if tag['name'] == git_commit_id: + if 'end_ts' not in tag: + # Active image tag + logging.info('Image ' + image + ':' + git_commit_id + + ' with this specific tag corresponding to the' + + ' commit id of the git HEAD exists on quay.io,' + + ' last modified on ' + + datetime.datetime.fromtimestamp(tag['start_ts']).isoformat() + + ' UTC') + return True + else: + # Tag used to exist + logging.error('Image ' + image + ':' + git_commit_id + + ' with this specific tag no longer exists on quay.io') + return False + except KeyError: + logging.error('Unable to parse response from quay.io (%s)' % res.url) + return 0 + logging.error('There is no image ' + image + ':' + git_commit_id + + ' with this specific tag on quay.io or it requires authentication') + return False + + # https://stackoverflow.com/a/14692747 def get_by_path(root, items): """Access a nested object in root by item sequence."""