airship
/
treasuremap
Code Issues Proposed changes

Merge "versions.yaml updater tool: verify that images & tags do exist"

This commit is contained in:
Zuul 2018-09-13 21:10:07 +00:00 committed by Gerrit Code Review
parent 25cedb0d8a 3efa98a31b
commit b7e9387b45
1 changed files with 104 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
tools/updater.py
View File

@ -23,12 +23,18 @@
# 2) tags of container images listed in dict `image_repo_git_url` in the
# code below
#
# In addition to that, the tool verifies that container images with
# specific tags equal to the HEAD git commit id's we reference, do really
# exist on quay.io repository and are available for download.
#
from functools import reduce
import argparse
import datetime
import logging
import operator
import os
import requests
import sys
try:
@ -54,7 +60,8 @@ image_repo_git_url = {
'quay.io/airshipit/airflow': 'https://git.openstack.org/openstack/airship-shipyard',
'quay.io/airshipit/armada': 'https://git.openstack.org/openstack/airship-armada',
'quay.io/airshipit/deckhand': 'https://git.openstack.org/openstack/airship-deckhand',
'quay.io/airshipit/divingbell': 'https://git.openstack.org/openstack/airship-divingbell',
# yes, divingbell image is just Ubuntu 16.04 image, and we don't check it's tag
#'docker.io/ubuntu': 'https://git.openstack.org/openstack/airship-divingbell',
'quay.io/airshipit/drydock': 'https://git.openstack.org/openstack/airship-drydock',
# maas-{rack,region}-controller images are built from airship-maas repository
'quay.io/airshipit/maas-rack-controller': 'https://git.openstack.org/openstack/airship-maas',
@ -64,18 +71,33 @@ image_repo_git_url = {
'quay.io/airshipit/shipyard': 'https://git.openstack.org/openstack/airship-shipyard',
# sstream-cache image is built from airship-maas repository
'quay.io/airshipit/sstream-cache': 'https://git.openstack.org/openstack/airship-maas',
'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios'
# Disabled by Kaspars: https://review.openstack.org/#/c/596909/21/tools/updater.py@53
#'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter'
'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios',
'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter'
}
logging.basicConfig(level=logging.INFO)
# Temporary dict of git url's and cached commit id's: {'git_url': 'commit_id'}
global git_url_commit_ids
git_url_commit_ids = {}
# Temporary dict of image repo's and status of image on quay.io
global image_repo_status
image_repo_status = {}
dict_path = None
def inverse_dict(dic):
"""Accepts dictionary, returns dictionary where keys become values,
and values become keys"""
new_dict = {}
for k, v in dic.items():
new_dict[v] = k
return new_dict
git_url_image_repo = inverse_dict(image_repo_git_url)
# https://stackoverflow.com/a/35585837
def lsremote(url, remote_ref):
"""Accepts git url and remote reference, returns git commit id."""
@ -92,14 +114,92 @@ def get_commit_id(url):
# If we don't have this git url in our url's dictionary,
# fetch latest commit ID and add new dictionary entry
logging.debug('git_url_commit_ids: %s', git_url_commit_ids)
logging.debug('image_repo_status: %s', image_repo_status)
if url not in git_url_commit_ids:
logging.debug('git url: ' + url +
' is not in git_url_commit_ids dict;' +
' adding it with HEAD commit id')
git_url_commit_ids[url] = lsremote(url, 'HEAD')
if url in git_url_image_repo:
if git_url_image_repo[url] not in image_repo_status:
image_repo_status[git_url_image_repo[url]] = \
verify_image_tag(git_url_image_repo[url], git_url_commit_ids[url])
else:
logging.debug('We checked image ' + git_url_image_repo[url] +
' on quay.io already, skipping')
return git_url_commit_ids[url]
def verify_image_tag(image, git_commit_id):
"""Verify if image with certain tag exists on quay.io,
returns 0 (image not hosted on quay.io), True, or False
"""
if not image.startswith('quay.io/'):
logging.info('Unable to verify if image ' + image + ':' +
git_commit_id + ' with this specific tag exists' +
' in containers repository: only quay.io is' +
' supported at the moment')
return 0
logging.info('Verifying if image ' + image + ':' + git_commit_id +
' with this specific tag exists on quay.io...')
retries = 0
max_retries = 5
payload = {'specificTag': git_commit_id}
hash_image = image.split('/')
url = 'https://quay.io/api/v1/repository/' + \
hash_image[1] + '/' + hash_image[2] + '/tag/'
while retries <= max_retries:
retries = retries +1
try:
res = requests.get(url, params = payload, timeout = 5)
except requests.exceptions.Timeout:
logging.warning("Failed to fetch url %s" % res.url)
sleep(1)
if retries == max_retries:
logging.error("Failed to connect to quay.io")
return 0
if res.status_code != 200:
logging.error('Image %s is not available on quay.io or it' +
' requires authentication', image)
return 0
try:
res = res.json()
except json.decoder.JSONDecodeError:
logging.error('Unable to parse response from quay.io (%s)' % res.url)
return 0
try:
for tag in res['tags']:
# Normally there should be only one tag description
if tag['name'] == git_commit_id:
if 'end_ts' not in tag:
# Active image tag
logging.info('Image ' + image + ':' + git_commit_id +
' with this specific tag corresponding to the' +
' commit id of the git HEAD exists on quay.io,' +
' last modified on ' +
datetime.datetime.fromtimestamp(tag['start_ts']).isoformat() +
' UTC')
return True
else:
# Tag used to exist
logging.error('Image ' + image + ':' + git_commit_id +
' with this specific tag no longer exists on quay.io')
return False
except KeyError:
logging.error('Unable to parse response from quay.io (%s)' % res.url)
return 0
logging.error('There is no image ' + image + ':' + git_commit_id +
' with this specific tag on quay.io or it requires authentication')
return False
# https://stackoverflow.com/a/14692747
def get_by_path(root, items):
"""Access a nested object in root by item sequence."""