diff --git a/test-requirements.txt b/test-requirements.txt index 6963e853d..d42264411 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -18,3 +18,4 @@ PyYAML>=3.12 # MIT reno>=2.5.0 # Apache-2.0 urllib3>=1.21.1 # MIT bashate>=0.2 # Apache-2.0 +requests-mock>=1.2.0 # Apache-2.0 diff --git a/tripleo_common/image/image_uploader.py b/tripleo_common/image/image_uploader.py index ab2c9b6d8..823560420 100644 --- a/tripleo_common/image/image_uploader.py +++ b/tripleo_common/image/image_uploader.py @@ -19,10 +19,12 @@ from concurrent import futures import json import netifaces import os +import re import requests +from requests import auth as requests_auth import shutil import six -from six.moves.urllib.parse import urlparse +from six.moves.urllib import parse import subprocess import tempfile import tenacity @@ -47,6 +49,7 @@ LOG = logging.getLogger(__name__) SECURE_REGISTRIES = ( 'trunk.registry.rdoproject.org', 'docker.io', + 'registry-1.docker.io', ) CLEANUP = ( @@ -84,10 +87,12 @@ class ImageUploadManager(BaseImageManager): self.dry_run = dry_run self.cleanup = cleanup - def discover_image_tag(self, image, tag_from_label=None): + def discover_image_tag(self, image, tag_from_label=None, + username=None, password=None): uploader = self.uploader(DEFAULT_UPLOADER) return uploader.discover_image_tag( - image, tag_from_label=tag_from_label) + image, tag_from_label=tag_from_label, + username=username, password=password) def uploader(self, uploader): if uploader not in self.uploaders: @@ -162,7 +167,8 @@ class ImageUploader(object): pass @abc.abstractmethod - def discover_image_tag(self, image, tag_from_label=None): + def discover_image_tag(self, image, tag_from_label=None, + username=None, password=None): """Discover a versioned tag for an image""" pass @@ -267,10 +273,10 @@ class BaseImageUploader(ImageUploader): 'Modifying image %s failed' % target_image) @staticmethod - def _images_match(image1, image2, insecure_registries): + def _images_match(image1, image2, insecure_registries, session1=None): try: image1_digest = BaseImageUploader._image_digest( - image1, insecure_registries) + image1, insecure_registries, session=session1) except Exception: return False try: @@ -285,27 +291,74 @@ class BaseImageUploader(ImageUploader): return image1_digest == image2_digest @staticmethod - def _image_digest(image, insecure_registries): + def _image_digest(image, insecure_registries, session=None): image_url = BaseImageUploader._image_to_url(image) insecure = image_url.netloc in insecure_registries - i = BaseImageUploader._inspect(image_url, insecure) + i = BaseImageUploader._inspect(image_url, insecure, session) return i.get('Digest') @staticmethod - def _image_labels(image, insecure): - image_url = BaseImageUploader._image_to_url(image) - i = BaseImageUploader._inspect(image_url, insecure) + def _image_labels(image_url, insecure, session=None): + i = BaseImageUploader._inspect(image_url, insecure, session) return i.get('Labels', {}) or {} @staticmethod - def _image_exists(image, insecure_registries): + def _image_exists(image, insecure_registries, session=None): try: - BaseImageUploader._image_digest(image, insecure_registries) + BaseImageUploader._image_digest( + image, insecure_registries, session=session) except ImageNotFoundException: return False else: return True + @staticmethod + def authenticate(image_url, username=None, password=None, insecure=False): + image_url = BaseImageUploader._fix_dockerio_url(image_url) + netloc = image_url.netloc + if insecure: + scheme = 'http' + else: + scheme = 'https' + image, tag = image_url.path.split(':') + url = '%s://%s/v2/' % (scheme, netloc) + session = requests.Session() + r = session.get(url, timeout=30) + LOG.debug('%s status code %s' % (url, r.status_code)) + if r.status_code != 401: + return session + if 'www-authenticate' not in r.headers: + raise ImageUploaderException( + 'Unknown authentication method for headers: %s' % r.headers) + + www_auth = r.headers['www-authenticate'] + if not www_auth.startswith('Bearer '): + raise ImageUploaderException( + 'Unknown www-authenticate value: %s' % www_auth) + token_param = {} + + realm = re.search('realm="(.*?)"', www_auth).group(1) + token_param['service'] = re.search( + 'service="(.*?)"', www_auth).group(1) + token_param['scope'] = 'repository:%s:pull' % image[1:] + auth = None + if username: + auth = requests_auth.HTTPBasicAuth(username, password) + rauth = session.get(realm, params=token_param, auth=auth, timeout=30) + rauth.raise_for_status() + session.headers['Authorization'] = 'Bearer %s' % rauth.json()['token'] + return session + + @staticmethod + def _fix_dockerio_url(url): + one = 'docker.io' + two = 'registry-1.docker.io' + if url.netloc != one: + return url + return parse.ParseResult(url.scheme, two, + url.path, url.params, + url.query, url.fragment) + @staticmethod @tenacity.retry( # Retry up to 5 times with jittered exponential backoff reraise=True, @@ -313,39 +366,79 @@ class BaseImageUploader(ImageUploader): wait=tenacity.wait_random_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(5) ) - def _inspect(image_url, insecure=False): - image = image_url.geturl() - - cmd = ['skopeo', 'inspect'] - + def _inspect(image_url, insecure=False, session=None): + original_image_url = image_url + image_url = BaseImageUploader._fix_dockerio_url(image_url) + parts = { + 'netloc': image_url.netloc + } if insecure: - cmd.append('--tls-verify=false') - cmd.append(image) + parts['scheme'] = 'http' + else: + parts['scheme'] = 'https' + image, tag = image_url.path.split(':') + parts['image'] = image + parts['tag'] = tag - LOG.info('Running %s' % ' '.join(cmd)) - env = os.environ.copy() - process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + manifest_url = ('%(scheme)s://%(netloc)s/v2' + '%(image)s/manifests/%(tag)s' % parts) + tags_url = ('%(scheme)s://%(netloc)s/v2' + '%(image)s/tags/list' % parts) + manifest_headers = { + 'Accept': 'application/vnd.docker.distribution.manifest.v2+json' + } - out, err = process.communicate() - if process.returncode != 0: - not_found_msgs = ( - u'manifest unknown', - # returned by docker.io - u'requested access to the resource is denied' - ) - if any(n in err for n in not_found_msgs): - raise ImageNotFoundException('Not found image: %s\n%s' % - (image, err)) - raise ImageUploaderException('Error inspecting image: %s\n%s' % - (image, err)) - return json.loads(out) + p = futures.ThreadPoolExecutor(max_workers=2) + manifest_f = p.submit( + session.get, manifest_url, headers=manifest_headers, timeout=30) + tags_f = p.submit(session.get, tags_url, timeout=30) + + manifest_r = manifest_f.result() + tags_r = tags_f.result() + + if manifest_r.status_code == 404: + raise ImageNotFoundException('Not found image: %s' % + image_url.geturl()) + manifest_r.raise_for_status() + tags_r.raise_for_status() + + manifest = manifest_r.json() + layers = [l['digest'] for l in manifest['layers']] + + parts['config_digest'] = manifest['config']['digest'] + config_headers = { + 'Accept': manifest['config']['mediaType'] + } + config_url = ('%(scheme)s://%(netloc)s/v2' + '%(image)s/blobs/%(config_digest)s' % parts) + config_f = p.submit( + session.get, config_url, headers=config_headers, timeout=30) + config_r = config_f.result() + config_r.raise_for_status() + + tags = tags_r.json()['tags'] + digest = manifest_r.headers['Docker-Content-Digest'] + config = config_r.json() + name = '%s%s' % (original_image_url.netloc, image) + + return { + 'Name': name, + 'Digest': digest, + 'RepoTags': tags, + 'Created': config['created'], + 'DockerVersion': config['docker_version'], + 'Labels': config['config']['Labels'], + 'Architecture': config['architecture'], + 'Os': config['os'], + 'Layers': layers, + } @staticmethod def _image_to_url(image): if '://' not in image: image = 'docker://' + image - return urlparse(image) + url = parse.urlparse(image) + return url @staticmethod def _discover_tag_from_inspect(i, image, tag_from_label=None, @@ -412,20 +505,25 @@ class BaseImageUploader(ImageUploader): return versioned_images def discover_image_tag(self, image, tag_from_label=None, - fallback_tag=None): + fallback_tag=None, username=None, password=None): image_url = self._image_to_url(image) insecure = self.is_insecure_registry(image_url.netloc) - i = self._inspect(image_url, insecure) + session = self.authenticate( + image_url, insecure=insecure, username=username, password=password) + i = self._inspect(image_url, insecure, session) return self._discover_tag_from_inspect(i, image, tag_from_label, fallback_tag) - def filter_images_with_labels(self, images, labels): + def filter_images_with_labels(self, images, labels, + username=None, password=None): images_with_labels = [] for image in images: url = self._image_to_url(image) - image_labels = self._image_labels(url.geturl(), - self.is_insecure_registry( - url.netloc)) + insecure = self.is_insecure_registry(url.netloc) + session = self.authenticate( + url, insecure=insecure, username=username, password=password) + image_labels = self._image_labels( + url, insecure=insecure, session=session) if set(labels).issubset(set(image_labels)): images_with_labels.append(image) @@ -465,7 +563,7 @@ class BaseImageUploader(ImageUploader): @staticmethod def _cross_repo_mount(target_image_url, image_layers, - source_layers, insecure_registries): + source_layers, insecure_registries, session): netloc = target_image_url.netloc name = target_image_url.path.split(':')[0][1:] if netloc in insecure_registries: @@ -483,7 +581,7 @@ class BaseImageUploader(ImageUploader): 'mount': layer, 'from': existing_name } - r = requests.post(url, data=data) + r = session.post(url, data=data) LOG.debug('%s %s' % (r.status_code, r.reason)) @@ -500,24 +598,34 @@ class DockerImageUploader(BaseImageUploader): source_tag = names['source_tag'] repo = names['repo'] source_image = names['source_image'] + source_image_url = BaseImageUploader._image_to_url(source_image) + source_insecure = source_image_url.netloc in insecure_registries target_image_no_tag = names['target_image_no_tag'] append_tag = names['append_tag'] target_tag = names['target_tag'] target_image_source_tag = names['target_image_source_tag'] target_image = names['target_image'] + target_image_url = BaseImageUploader._image_to_url(target_image) + target_insecure = target_image_url.netloc in insecure_registries if dry_run: return [] if modify_role: + target_session = BaseImageUploader.authenticate( + target_image_url, insecure=target_insecure) if BaseImageUploader._image_exists(target_image, - insecure_registries): + insecure_registries, + session=target_session): LOG.warning('Skipping upload for modified image %s' % target_image) return [] else: + source_session = BaseImageUploader.authenticate( + source_image_url, insecure=source_insecure) if BaseImageUploader._images_match(source_image, target_image, - insecure_registries): + insecure_registries, + session1=source_session): LOG.warning('Skipping upload for image %s' % image_name) return [] @@ -631,29 +739,40 @@ class SkopeoImageUploader(BaseImageUploader): source_image = names['source_image'] source_image_url = BaseImageUploader._image_to_url(source_image) - source_image_local_url = urlparse('containers-storage:%s' - % source_image) + source_image_local_url = parse.urlparse('containers-storage:%s' + % source_image) + source_insecure = source_image_url.netloc in insecure_registries + append_tag = names['append_tag'] target_image_source_tag = names['target_image_source_tag'] target_image = names['target_image'] target_image_url = BaseImageUploader._image_to_url(target_image) - target_image_local_url = urlparse('containers-storage:%s' % - target_image) + target_image_local_url = parse.urlparse('containers-storage:%s' % + target_image) + target_insecure = target_image_local_url.netloc in insecure_registries if dry_run: return [] + target_session = BaseImageUploader.authenticate( + target_image_url, insecure=target_insecure) + if modify_role and BaseImageUploader._image_exists( - target_image, insecure_registries): + target_image, insecure_registries, target_session): LOG.warning('Skipping upload for modified image %s' % target_image) return [] - source_inspect = BaseImageUploader._inspect(source_image_url) + source_session = BaseImageUploader.authenticate( + source_image_url, insecure=source_insecure) + + source_inspect = BaseImageUploader._inspect( + source_image_url, insecure=source_insecure, session=source_session) source_layers = source_inspect.get('Layers', []) BaseImageUploader._cross_repo_mount( - target_image_url, image_layers, source_layers, insecure_registries) + target_image_url, image_layers, source_layers, insecure_registries, + session=source_session) to_cleanup = [] if modify_role: @@ -671,8 +790,6 @@ class SkopeoImageUploader(BaseImageUploader): modify_role, modify_vars, source_image, target_image_source_tag, append_tag, container_build_tool='buildah') - # Inspect to confirm the playbook created the target image - BaseImageUploader._inspect(target_image_local_url) if cleanup == CLEANUP_FULL: to_cleanup.append(target_image) @@ -756,7 +873,7 @@ class SkopeoImageUploader(BaseImageUploader): if not image: continue LOG.warning('Removing local copy of %s' % image) - image_url = urlparse('containers-storage:%s' % image) + image_url = parse.urlparse('containers-storage:%s' % image) SkopeoImageUploader._delete(image_url) def run_tasks(self): @@ -795,7 +912,9 @@ def discover_tag_from_inspect(args): image, tag_from_label, insecure_registries = args image_url = BaseImageUploader._image_to_url(image) insecure = image_url.netloc in insecure_registries - i = BaseImageUploader._inspect(image_url, insecure) + session = BaseImageUploader.authenticate(image_url, insecure=insecure) + i = BaseImageUploader._inspect(image_url, insecure=insecure, + session=session) if ':' in image_url.path: # break out the tag from the url to be the fallback tag path = image.rpartition(':') diff --git a/tripleo_common/tests/image/test_image_uploader.py b/tripleo_common/tests/image/test_image_uploader.py index c3dbd559a..ca943eb4d 100644 --- a/tripleo_common/tests/image/test_image_uploader.py +++ b/tripleo_common/tests/image/test_image_uploader.py @@ -13,11 +13,11 @@ # under the License. # -import json import mock import operator import os import requests +from requests_mock.contrib import fixture as rm_fixture import six from six.moves.urllib.parse import urlparse import tempfile @@ -50,6 +50,8 @@ class TestImageUploadManager(base.TestCase): files.append('testfile') self.filelist = files + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader.authenticate') @mock.patch('tripleo_common.image.image_uploader.' 'BaseImageUploader._inspect') @mock.patch('tripleo_common.image.base.open', @@ -66,7 +68,8 @@ class TestImageUploadManager(base.TestCase): @mock.patch('tripleo_common.image.image_uploader.' 'get_undercloud_registry', return_value='192.0.2.0:8787') def test_file_parsing(self, mock_gur, mockdocker, mockioctl, mockpath, - mock_images_match, mock_is_insecure, mock_inspect): + mock_images_match, mock_is_insecure, mock_inspect, + mock_auth): mock_inspect.return_value = {} manager = image_uploader.ImageUploadManager(self.filelist, debug=True) @@ -171,89 +174,101 @@ class TestBaseImageUploader(base.TestCase): super(TestBaseImageUploader, self).setUp() self.uploader = image_uploader.BaseImageUploader() self.uploader._inspect.retry.sleep = mock.Mock() + self.requests = self.useFixture(rm_fixture.Fixture()) - @mock.patch('requests.get') - def test_is_insecure_registry_known(self, mock_get): + def test_is_insecure_registry_known(self): self.assertFalse( self.uploader.is_insecure_registry('docker.io')) - @mock.patch('requests.get') - def test_is_insecure_registry_secure(self, mock_get): + def test_is_insecure_registry_secure(self): self.assertFalse( self.uploader.is_insecure_registry('192.0.2.0:8787')) self.assertFalse( self.uploader.is_insecure_registry('192.0.2.0:8787')) - mock_get.assert_called_once_with('https://192.0.2.0:8787/') + self.assertEqual( + 'https://192.0.2.0:8787/', + self.requests.request_history[0].url + ) - @mock.patch('requests.get') - def test_is_insecure_registry_timeout(self, mock_get): - mock_get.side_effect = requests.exceptions.ReadTimeout('ouch') + def test_is_insecure_registry_timeout(self): + self.requests.get( + 'https://192.0.2.0:8787/', + exc=requests.exceptions.ReadTimeout('ouch')) self.assertFalse( self.uploader.is_insecure_registry('192.0.2.0:8787')) self.assertFalse( self.uploader.is_insecure_registry('192.0.2.0:8787')) - mock_get.assert_called_once_with('https://192.0.2.0:8787/') + self.assertEqual( + 'https://192.0.2.0:8787/', + self.requests.request_history[0].url + ) - @mock.patch('requests.get') - def test_is_insecure_registry_insecure(self, mock_get): - mock_get.side_effect = requests.exceptions.SSLError('ouch') + def test_is_insecure_registry_insecure(self): + self.requests.get( + 'https://192.0.2.0:8787/', + exc=requests.exceptions.SSLError('ouch')) self.assertTrue( self.uploader.is_insecure_registry('192.0.2.0:8787')) self.assertTrue( self.uploader.is_insecure_registry('192.0.2.0:8787')) - mock_get.assert_called_once_with('https://192.0.2.0:8787/') + self.assertEqual( + 'https://192.0.2.0:8787/', + self.requests.request_history[0].url + ) - @mock.patch('subprocess.Popen') - def test_discover_image_tag(self, mock_popen): - result = { + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader.authenticate') + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader._inspect') + def test_discover_image_tag(self, mock_inspect, mock_auth): + mock_inspect.return_value = { 'Labels': { 'rdo_version': 'a', 'build_version': '4.0.0' }, 'RepoTags': ['a'] } - mock_process = mock.Mock() - mock_process.communicate.return_value = (json.dumps(result), '') - mock_process.returncode = 0 - mock_popen.return_value = mock_process self.assertEqual( 'a', - self.uploader.discover_image_tag('docker.io/t/foo', 'rdo_version') + self.uploader.discover_image_tag('docker.io/t/foo:b', + 'rdo_version') ) # no tag_from_label specified self.assertRaises( ImageUploaderException, self.uploader.discover_image_tag, - 'docker.io/t/foo') + 'docker.io/t/foo:b') # missing RepoTags entry self.assertRaises( ImageUploaderException, self.uploader.discover_image_tag, - 'docker.io/t/foo', + 'docker.io/t/foo:b', 'build_version') # missing Labels entry self.assertRaises( ImageUploaderException, self.uploader.discover_image_tag, - 'docker.io/t/foo', + 'docker.io/t/foo:b', 'version') # inspect call failed - mock_process.returncode = 1 - mock_process.communicate.return_value = ('', 'manifest unknown') + mock_inspect.side_effect = ImageNotFoundException() self.assertRaises( ImageNotFoundException, self.uploader.discover_image_tag, - 'docker.io/t/foo', + 'docker.io/t/foo:b', 'rdo_version') - @mock.patch('subprocess.Popen') - def test_discover_tag_from_inspect(self, mock_popen): - result = { + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader.authenticate') + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader._inspect') + def test_discover_tag_from_inspect(self, mock_inspect, mock_auth): + mock_inspect.return_value = { 'Labels': { 'rdo_version': 'a', 'build_version': '4.0.0', @@ -262,10 +277,6 @@ class TestBaseImageUploader(base.TestCase): }, 'RepoTags': ['a', '1.0.0-20180125'] } - mock_process = mock.Mock() - mock_process.communicate.return_value = (json.dumps(result), '') - mock_process.returncode = 0 - mock_popen.return_value = mock_process sr = image_uploader.SECURE_REGISTRIES # simple label -> tag @@ -332,7 +343,7 @@ class TestBaseImageUploader(base.TestCase): ) # inspect call failed - mock_process.returncode = 1 + mock_inspect.side_effect = ImageUploaderException() self.assertRaises( ImageUploaderException, image_uploader.discover_tag_from_inspect, @@ -388,6 +399,119 @@ class TestBaseImageUploader(base.TestCase): mock_inspect.side_effect = ImageUploaderException() self.assertFalse(self.uploader._images_match('foo', 'bar', set())) + def test_authenticate(self): + req = self.requests + auth = image_uploader.BaseImageUploader.authenticate + url1 = urlparse('docker://docker.io/t/nova-api:latest') + + # no auth required + req.get('https://registry-1.docker.io/v2/', status_code=200) + self.assertNotIn('Authorization', auth(url1).headers) + + # missing 'www-authenticate' header + req.get('https://registry-1.docker.io/v2/', status_code=401) + self.assertRaises(ImageUploaderException, auth, url1) + + # unknown 'www-authenticate' header + req.get('https://registry-1.docker.io/v2/', status_code=401, + headers={'www-authenticate': 'Foo'}) + self.assertRaises(ImageUploaderException, auth, url1) + + # successful auth requests + headers = { + 'www-authenticate': 'Bearer ' + 'realm="https://auth.docker.io/token",' + 'service="registry.docker.io"' + } + req.get('https://registry-1.docker.io/v2/', status_code=401, + headers=headers) + req.get('https://auth.docker.io/token', json={"token": "asdf1234"}) + self.assertEqual( + 'Bearer asdf1234', + auth(url1).headers['Authorization'] + ) + + def test_fix_dockerio_url(self): + url1 = urlparse('docker://docker.io/t/nova-api:latest') + url2 = urlparse('docker://registry-1.docker.io/t/nova-api:latest') + url3 = urlparse('docker://192.0.2.1:8787/t/nova-api:latest') + fix = image_uploader.BaseImageUploader._fix_dockerio_url + # fix urls + self.assertEqual(url2, fix(url1)) + + # no change urls + self.assertEqual(url2, fix(url2)) + self.assertEqual(url3, fix(url3)) + + def test_inspect(self): + req = self.requests + session = requests.Session() + session.headers['Authorization'] = 'Bearer asdf1234' + inspect = image_uploader.BaseImageUploader._inspect + + url1 = urlparse('docker://docker.io/t/nova-api:latest') + + manifest_resp = { + 'config': { + 'mediaType': 'text/html', + 'digest': 'abcdef' + }, + 'layers': [ + {'digest': 'aaa'}, + {'digest': 'bbb'}, + {'digest': 'ccc'}, + ] + } + manifest_headers = {'Docker-Content-Digest': 'eeeeee'} + tags_resp = {'tags': ['one', 'two', 'latest']} + config_resp = { + 'created': '2018-10-02T11:13:45.567533229Z', + 'docker_version': '1.13.1', + 'config': { + 'Labels': { + 'build-date': '20181002', + 'build_id': '1538477701', + 'kolla_version': '7.0.0' + } + }, + 'architecture': 'amd64', + 'os': 'linux', + } + + req.get('https://registry-1.docker.io/v2/t/nova-api/tags/list', + json=tags_resp) + req.get('https://registry-1.docker.io/v2/t/nova-api/blobs/abcdef', + json=config_resp) + + # test 404 response + req.get('https://registry-1.docker.io/v2/t/nova-api/manifests/latest', + status_code=404) + self.assertRaises(ImageNotFoundException, inspect, url1, + session=session) + + # test full response + req.get('https://registry-1.docker.io/v2/t/nova-api/manifests/latest', + json=manifest_resp, headers=manifest_headers) + + self.assertEqual( + { + 'Architecture': 'amd64', + 'Created': '2018-10-02T11:13:45.567533229Z', + 'Digest': 'eeeeee', + 'DockerVersion': '1.13.1', + 'Labels': { + 'build-date': '20181002', + 'build_id': '1538477701', + 'kolla_version': '7.0.0' + }, + 'Layers': ['aaa', 'bbb', 'ccc'], + 'Name': 'docker.io/t/nova-api', + 'Os': 'linux', + 'RepoTags': ['one', 'two', 'latest'] + }, + inspect(url1, session=session) + ) + class TestDockerImageUploader(base.TestCase): @@ -405,22 +529,18 @@ class TestDockerImageUploader(base.TestCase): super(TestDockerImageUploader, self).tearDown() self.patcher.stop() - @mock.patch('subprocess.Popen') - def test_upload_image(self, mock_popen): + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader.authenticate') + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader._inspect') + def test_upload_image(self, mock_inspect, mock_auth): result1 = { 'Digest': 'a' } result2 = { 'Digest': 'b' } - mock_process = mock.Mock() - mock_process.communicate.side_effect = [ - (json.dumps(result1), ''), - (json.dumps(result2), ''), - ] - - mock_process.returncode = 0 - mock_popen.return_value = mock_process + mock_inspect.side_effect = [result1, result2] image = 'docker.io/tripleomaster/heat-docker-agents-centos' tag = 'latest' @@ -457,46 +577,14 @@ class TestDockerImageUploader(base.TestCase): push_image, tag=tag, stream=True) - @mock.patch('subprocess.Popen') - def test_upload_image_missing_tag(self, mock_popen): - image = 'docker.io/tripleomaster/heat-docker-agents-centos' - expected_tag = 'latest' - push_destination = 'localhost:8787' - push_image = 'localhost:8787/tripleomaster/heat-docker-agents-centos' - - self.uploader.upload_image(image, - None, - push_destination, - set(), - None, - None, - None, - False, - 'full', - {}) - - self.dockermock.assert_called_once_with( - base_url='unix://var/run/docker.sock', version='auto') - - self.dockermock.return_value.pull.assert_called_once_with( - image, tag=expected_tag, stream=True) - self.dockermock.return_value.tag.assert_called_once_with( - image=image + ':' + expected_tag, - repository=push_image, - tag=expected_tag, force=True) - self.dockermock.return_value.push.assert_called_once_with( - push_image, - tag=expected_tag, stream=True) - - @mock.patch('subprocess.Popen') - def test_upload_image_existing(self, mock_popen): - result = { + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader.authenticate') + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader._inspect') + def test_upload_image_existing(self, mock_inspect, mock_auth): + mock_inspect.return_value = { 'Digest': 'a' } - mock_process = mock.Mock() - mock_process.communicate.return_value = (json.dumps(result), '') - mock_process.returncode = 0 - mock_popen.return_value = mock_process image = 'docker.io/tripleomaster/heat-docker-agents-centos' tag = 'latest' push_destination = 'localhost:8787' @@ -523,16 +611,14 @@ class TestDockerImageUploader(base.TestCase): self.dockermock.return_value.tag.assert_not_called() self.dockermock.return_value.push.assert_not_called() - @mock.patch('subprocess.Popen') + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader.authenticate') + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader._inspect') @mock.patch('tripleo_common.actions.' 'ansible.AnsiblePlaybookAction', autospec=True) - def test_modify_upload_image(self, mock_ansible, mock_popen): - mock_process = mock.Mock() - mock_process.communicate.return_value = ( - '', 'FATA[0000] Error reading manifest: manifest unknown') - - mock_process.returncode = 1 - mock_popen.return_value = mock_process + def test_modify_upload_image(self, mock_ansible, mock_inspect, mock_auth): + mock_inspect.side_effect = ImageNotFoundException() mock_ansible.return_value.run.return_value = {} image = 'docker.io/tripleomaster/heat-docker-agents-centos' @@ -591,15 +677,14 @@ class TestDockerImageUploader(base.TestCase): tag=tag + append_tag, stream=True) - @mock.patch('subprocess.Popen') + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader.authenticate') + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader._inspect') @mock.patch('tripleo_common.actions.' 'ansible.AnsiblePlaybookAction', autospec=True) - def test_modify_image_failed(self, mock_ansible, mock_popen): - mock_process = mock.Mock() - mock_process.communicate.return_value = ('', 'manifest unknown') - - mock_process.returncode = 1 - mock_popen.return_value = mock_process + def test_modify_image_failed(self, mock_ansible, mock_inspect, mock_auth): + mock_inspect.side_effect = ImageNotFoundException() image = 'docker.io/tripleomaster/heat-docker-agents-centos' tag = 'latest' @@ -655,11 +740,14 @@ class TestDockerImageUploader(base.TestCase): mock_process.communicate.assert_not_called() self.assertEqual([], result) + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader.authenticate') @mock.patch('tripleo_common.image.image_uploader.' 'BaseImageUploader._inspect') @mock.patch('tripleo_common.actions.' 'ansible.AnsiblePlaybookAction', autospec=True) - def test_modify_image_existing(self, mock_ansible, mock_inspect): + def test_modify_image_existing(self, mock_ansible, mock_inspect, + mock_auth): mock_inspect.return_value = {'Digest': 'a'} image = 'docker.io/tripleomaster/heat-docker-agents-centos' @@ -772,7 +860,10 @@ class TestSkopeoImageUploader(base.TestCase): @mock.patch('subprocess.Popen') @mock.patch('tripleo_common.image.image_uploader.' 'BaseImageUploader._inspect') - def test_upload_image(self, mock_inspect, mock_popen, mock_environ): + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader.authenticate') + def test_upload_image(self, mock_auth, mock_inspect, + mock_popen, mock_environ): mock_process = mock.Mock() mock_process.communicate.return_value = ('copy complete', '') mock_process.returncode = 0 @@ -807,6 +898,8 @@ class TestSkopeoImageUploader(base.TestCase): env={}, stdout=-1 ) + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader.authenticate') @mock.patch('tripleo_common.image.image_uploader.' 'BaseImageUploader._inspect') @mock.patch('tripleo_common.image.image_uploader.' @@ -816,7 +909,7 @@ class TestSkopeoImageUploader(base.TestCase): @mock.patch('tripleo_common.actions.' 'ansible.AnsiblePlaybookAction', autospec=True) def test_modify_upload_image(self, mock_ansible, mock_exists, mock_copy, - mock_inspect): + mock_inspect, mock_auth): mock_exists.return_value = False mock_inspect.return_value = {} with tempfile.NamedTemporaryFile(delete=False) as logfile: @@ -868,10 +961,7 @@ class TestSkopeoImageUploader(base.TestCase): mock_inspect.assert_has_calls([ mock.call(urlparse( 'docker://docker.io/t/nova-api:latest' - )), - mock.call(urlparse( - 'containers-storage:localhost:8787/t/nova-api:latestmodify-123' - )) + ), insecure=False, session=mock.ANY) ]) mock_copy.assert_has_calls([ mock.call( @@ -894,6 +984,8 @@ class TestSkopeoImageUploader(base.TestCase): extra_env_variables=mock.ANY ) + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader.authenticate') @mock.patch('tripleo_common.image.image_uploader.' 'BaseImageUploader._inspect') @mock.patch('tripleo_common.image.image_uploader.' @@ -903,7 +995,7 @@ class TestSkopeoImageUploader(base.TestCase): @mock.patch('tripleo_common.actions.' 'ansible.AnsiblePlaybookAction', autospec=True) def test_modify_image_failed(self, mock_ansible, mock_exists, mock_copy, - mock_inspect): + mock_inspect, mock_auth): mock_exists.return_value = False mock_inspect.return_value = {} @@ -959,11 +1051,14 @@ class TestSkopeoImageUploader(base.TestCase): mock_process.communicate.assert_not_called() self.assertEqual([], result) + @mock.patch('tripleo_common.image.image_uploader.' + 'BaseImageUploader.authenticate') @mock.patch('tripleo_common.image.image_uploader.' 'BaseImageUploader._inspect') @mock.patch('tripleo_common.actions.' 'ansible.AnsiblePlaybookAction', autospec=True) - def test_modify_image_existing(self, mock_ansible, mock_inspect): + def test_modify_image_existing(self, mock_ansible, mock_inspect, + mock_auth): mock_inspect.return_value = {'Digest': 'a'} image = 'docker.io/t/nova-api'