From 589ac8ac0d5a14ce4aa831892d3b0a4a31e29674 Mon Sep 17 00:00:00 2001 From: James Slagle Date: Mon, 6 Jul 2020 15:16:02 -0400 Subject: [PATCH] Don't assume default tag exists in container repo When no tag is set for an entry in ContainerImagePrepare, the default tag from container-images/container_image_prepare_defaults.yaml will be assumed, which is typically current-tripleo, or a release version such as 16.0. In most cases, this tag will exist. However, when using a satellite with a content view that has been filtered on tags, it likely won't exist since content views are often used with container image versions that are not the latest. In the case where no tag is set in the entry in ContainerImagePrepare, and the default tag does not exist in the container repo, the latest tag from the repo will be assumed instead. Change-Id: I985ef22c340c4071866c8c51bf303a6f4ee7713c Closes-Bug: #1886547 Signed-off-by: James Slagle (cherry picked from commit 0c4de9c771fad75ebabe2d18f5aef4b76613d52a) (cherry picked from commit ec65b686c38f696ac2aefc9d7dc9376f5637aa1d) --- .../container_image_prepare_defaults.yaml | 1 + ...heck_for_default_tag-09fe34d2ac434890.yaml | 6 ++++ tripleo_common/image/image_uploader.py | 31 ++++++++-------- tripleo_common/image/kolla_builder.py | 12 ++++++- .../tests/image/test_image_uploader.py | 36 ++++++++++--------- .../tests/image/test_kolla_builder.py | 2 ++ 6 files changed, 56 insertions(+), 32 deletions(-) create mode 100644 releasenotes/notes/check_for_default_tag-09fe34d2ac434890.yaml diff --git a/container-images/container_image_prepare_defaults.yaml b/container-images/container_image_prepare_defaults.yaml index 6a399791f..f7841208b 100644 --- a/container-images/container_image_prepare_defaults.yaml +++ b/container-images/container_image_prepare_defaults.yaml @@ -17,6 +17,7 @@ parameter_defaults: name_prefix: centos-binary- name_suffix: '' tag: current-tripleo + default_tag: True rhel_containers: false # Substitute neutron images based on driver. Can be 'other', 'ovn' or diff --git a/releasenotes/notes/check_for_default_tag-09fe34d2ac434890.yaml b/releasenotes/notes/check_for_default_tag-09fe34d2ac434890.yaml new file mode 100644 index 000000000..6bdf019b9 --- /dev/null +++ b/releasenotes/notes/check_for_default_tag-09fe34d2ac434890.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - When the default tag doesn't exist in the container repo during container + image prepare, and a tag wasn't set in the actual input for + ContainerImagePrepare, the latest tag from the repo will be used instead of + failing with a not found error. diff --git a/tripleo_common/image/image_uploader.py b/tripleo_common/image/image_uploader.py index 6a03f293a..78a71df62 100644 --- a/tripleo_common/image/image_uploader.py +++ b/tripleo_common/image/image_uploader.py @@ -695,19 +695,24 @@ class BaseImageUploader(object): wait=tenacity.wait_random_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(5) ) - def _inspect(cls, image_url, session=None): + def _inspect(cls, image_url, session=None, default_tag=False): image, tag = cls._image_tag_from_url(image_url) parts = { 'image': image, 'tag': tag } - manifest_url = cls._build_url( - image_url, CALL_MANIFEST % parts - ) tags_url = cls._build_url( image_url, CALL_TAGS % parts ) + tags_r = RegistrySessionHelper.get(session, tags_url, timeout=30) + tags = tags_r.json()['tags'] + if default_tag and tag not in tags: + parts['tag'] = tags[-1] + + manifest_url = cls._build_url( + image_url, CALL_MANIFEST % parts + ) manifest_headers = {'Accept': MEDIA_MANIFEST_V2} try: @@ -724,8 +729,6 @@ class BaseImageUploader(object): else: raise - tags_r = RegistrySessionHelper.get(session, tags_url, timeout=30) - manifest_str = cls._get_response_text(manifest_r) if 'Docker-Content-Digest' in manifest_r.headers: @@ -759,8 +762,6 @@ class BaseImageUploader(object): ) config = config_r.json() - tags = tags_r.json()['tags'] - image, tag = cls._image_tag_from_url(image_url) name = '%s%s' % (image_url.netloc, image) created = config['created'] @@ -910,7 +911,8 @@ class BaseImageUploader(object): ) return tag_label - def discover_image_tags(self, images, tag_from_label=None): + def discover_image_tags(self, images, tag_from_label=None, + default_tag=False): image_urls = [self._image_to_url(i) for i in images] # prime self.insecure_registries by testing every image @@ -919,7 +921,8 @@ class BaseImageUploader(object): discover_args = [] for image in images: - discover_args.append((self, image, tag_from_label)) + discover_args.append((self, image, tag_from_label, + default_tag)) versioned_images = {} with futures.ThreadPoolExecutor(max_workers=16) as p: @@ -2326,10 +2329,10 @@ class PythonImageUploader(BaseImageUploader): return image, manifest, config_str @classmethod - def _inspect(cls, image_url, session=None): + def _inspect(cls, image_url, session=None, default_tag=False): if image_url.scheme == 'docker': return super(PythonImageUploader, cls)._inspect( - image_url, session=session) + image_url, session=session, default_tag=default_tag) if image_url.scheme != 'containers-storage': raise ImageUploaderException('Inspect not implemented for %s' % image_url.geturl()) @@ -2490,7 +2493,7 @@ def upload_task(args): def discover_tag_from_inspect(args): - self, image, tag_from_label = args + self, image, tag_from_label, default_tag = args image_url = self._image_to_url(image) username, password = self.credentials_for_registry(image_url.netloc) try: @@ -2503,7 +2506,7 @@ def discover_tag_from_inspect(args): 'missing registry credentials or the provided ' 'container or namespace does not exist. %s' % e) raise - i = self._inspect(image_url, session=session) + i = self._inspect(image_url, session=session, default_tag=default_tag) session.close() if ':' in image_url.path: # break out the tag from the url to be the fallback tag diff --git a/tripleo_common/image/kolla_builder.py b/tripleo_common/image/kolla_builder.py index 20c8088ef..59074ad06 100644 --- a/tripleo_common/image/kolla_builder.py +++ b/tripleo_common/image/kolla_builder.py @@ -330,10 +330,14 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE, ) uploader = manager.uploader('python') images = [i.get('imagename', '') for i in result] + if result: + default_tag = result[0].get('default_tag', False) + else: + default_tag = False if tag_from_label: image_version_tags = uploader.discover_image_tags( - images, tag_from_label) + images, tag_from_label, default_tag) for entry in result: imagename = entry.get('imagename', '') image_no_tag = imagename.rpartition(':')[0] @@ -438,6 +442,12 @@ class KollaImageBuilder(base.BaseImageManager): hyphenated appropriately. ''' mapping = dict(kwargs) + # set a flag to record whether the default tag is used or not. the + # logic here is that if the tag key is not already in mapping then it + # wil be added during the template render, so default_tag is set to + # True. + mapping['default_tag'] = 'tag' not in mapping + if CONTAINER_IMAGES_DEFAULTS is None: return for k, v in CONTAINER_IMAGES_DEFAULTS.items(): diff --git a/tripleo_common/tests/image/test_image_uploader.py b/tripleo_common/tests/image/test_image_uploader.py index 4991083d2..bd06eba76 100644 --- a/tripleo_common/tests/image/test_image_uploader.py +++ b/tripleo_common/tests/image/test_image_uploader.py @@ -527,12 +527,12 @@ class TestBaseImageUploader(base.TestCase): self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - (self.uploader, 'docker.io/t/foo', 'rdo_version') + (self.uploader, 'docker.io/t/foo', 'rdo_version', False) ) self.assertRaises( requests.exceptions.HTTPError, image_uploader.discover_tag_from_inspect, - (self.uploader, 'docker.io/t/foo', 'rdo_version') + (self.uploader, 'docker.io/t/foo', 'rdo_version', False) ) @mock.patch('tripleo_common.image.image_uploader.' @@ -554,63 +554,65 @@ class TestBaseImageUploader(base.TestCase): self.assertEqual( ('docker.io/t/foo', 'a'), image_uploader.discover_tag_from_inspect( - (self.uploader, 'docker.io/t/foo', 'rdo_version')) + (self.uploader, 'docker.io/t/foo', 'rdo_version', False)) ) # templated labels -> tag self.assertEqual( ('docker.io/t/foo', '1.0.0-20180125'), image_uploader.discover_tag_from_inspect( - (self.uploader, 'docker.io/t/foo', '{release}-{version}')) + (self.uploader, 'docker.io/t/foo', '{release}-{version}', + False)) ) # simple label -> tag with fallback self.assertEqual( ('docker.io/t/foo', 'a'), image_uploader.discover_tag_from_inspect( - (self.uploader, 'docker.io/t/foo:a', 'bar')) + (self.uploader, 'docker.io/t/foo:a', 'bar', False)) ) # templated labels -> tag with fallback self.assertEqual( ('docker.io/t/foo', 'a'), image_uploader.discover_tag_from_inspect( - (self.uploader, 'docker.io/t/foo:a', '{releases}-{versions}')) + (self.uploader, 'docker.io/t/foo:a', '{releases}-{versions}', + False)) ) # Invalid template self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - (self.uploader, 'docker.io/t/foo', '{release}-{version') + (self.uploader, 'docker.io/t/foo', '{release}-{version', False) ) # Missing label in template self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - (self.uploader, 'docker.io/t/foo', '{releases}-{version}') + (self.uploader, 'docker.io/t/foo', '{releases}-{version}', False) ) # no tag_from_label specified self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - (self.uploader, 'docker.io/t/foo', None) + (self.uploader, 'docker.io/t/foo', None, False) ) # missing RepoTags entry self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - (self.uploader, 'docker.io/t/foo', 'build_version') + (self.uploader, 'docker.io/t/foo', 'build_version', False) ) # missing Labels entry self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - (self.uploader, 'docker.io/t/foo', 'version') + (self.uploader, 'docker.io/t/foo', 'version', False) ) # inspect call failed @@ -618,7 +620,7 @@ class TestBaseImageUploader(base.TestCase): self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - (self.uploader, 'docker.io/t/foo', 'rdo_version') + (self.uploader, 'docker.io/t/foo', 'rdo_version', False) ) # handle auth issues @@ -632,12 +634,12 @@ class TestBaseImageUploader(base.TestCase): self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, - (self.uploader, 'docker.io/t/foo', 'rdo_version') + (self.uploader, 'docker.io/t/foo', 'rdo_version', False) ) self.assertRaises( requests.exceptions.HTTPError, image_uploader.discover_tag_from_inspect, - (self.uploader, 'docker.io/t/foo', 'rdo_version') + (self.uploader, 'docker.io/t/foo', 'rdo_version', False) ) @mock.patch('concurrent.futures.ThreadPoolExecutor') @@ -665,9 +667,9 @@ class TestBaseImageUploader(base.TestCase): mock_map.assert_called_once_with( image_uploader.discover_tag_from_inspect, [ - (self.uploader, 'docker.io/t/foo', 'rdo_release'), - (self.uploader, 'docker.io/t/bar', 'rdo_release'), - (self.uploader, 'docker.io/t/baz', 'rdo_release') + (self.uploader, 'docker.io/t/foo', 'rdo_release', False), + (self.uploader, 'docker.io/t/bar', 'rdo_release', False), + (self.uploader, 'docker.io/t/baz', 'rdo_release', False) ]) @mock.patch('tripleo_common.image.image_uploader.' diff --git a/tripleo_common/tests/image/test_kolla_builder.py b/tripleo_common/tests/image/test_kolla_builder.py index 89878931b..cd780401a 100644 --- a/tripleo_common/tests/image/test_kolla_builder.py +++ b/tripleo_common/tests/image/test_kolla_builder.py @@ -254,6 +254,7 @@ class TestKollaImageBuilderTemplate(base.TestCase): 'tag': 'current-tripleo', 'rhel_containers': False, 'neutron_driver': 'ovn', + 'default_tag': True, }, builder.container_images_template_inputs() ) @@ -281,6 +282,7 @@ class TestKollaImageBuilderTemplate(base.TestCase): 'tag': 'master', 'rhel_containers': False, 'neutron_driver': 'ovn', + 'default_tag': False, }, builder.container_images_template_inputs( namespace='192.0.2.0:5000/tripleotrain',