openstack
/
tripleo-common
Code Issues Proposed changes

Replace skopeo inspect with python 

This replaces the skopeo inspect calls with python equivalent. It is
faster than the skopeo inspect for two reasons:
- the auth token is shared for all requests
- the tags list request is made concurrently

This should also help with running the dry-run prepare in the mistral
podman container since /run is not involved at all in this
implementation.

Change-Id: Ia898d0acfdeac1699e7e08e2935a2a4eaf578531
Closes-Bug: #1797114
This commit is contained in:
Steve Baker 2018-10-11 17:38:50 +13:00 committed by Emilien Macchi
parent b4053ad111
commit 28a5ba0a01
3 changed files with 383 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test-requirements.txt
View File

@ -18,3 +18,4 @@ PyYAML>=3.12 # MIT
reno>=2.5.0 # Apache-2.0
urllib3>=1.21.1 # MIT
bashate>=0.2 # Apache-2.0
requests-mock>=1.2.0 # Apache-2.0

237
tripleo_common/image/image_uploader.py
View File

@ -19,10 +19,12 @@ from concurrent import futures
import json
import netifaces
import os
import re
import requests
from requests import auth as requests_auth
import shutil
import six
from six.moves.urllib.parse import urlparse
from six.moves.urllib import parse
import subprocess
import tempfile
import tenacity
@ -47,6 +49,7 @@ LOG = logging.getLogger(__name__)
SECURE_REGISTRIES = (
'trunk.registry.rdoproject.org',
'docker.io',
'registry-1.docker.io',
)
CLEANUP = (
@ -84,10 +87,12 @@ class ImageUploadManager(BaseImageManager):
self.dry_run = dry_run
self.cleanup = cleanup
def discover_image_tag(self, image, tag_from_label=None):
def discover_image_tag(self, image, tag_from_label=None,
username=None, password=None):
uploader = self.uploader(DEFAULT_UPLOADER)
return uploader.discover_image_tag(
image, tag_from_label=tag_from_label)
image, tag_from_label=tag_from_label,
username=username, password=password)
def uploader(self, uploader):
if uploader not in self.uploaders:
@ -162,7 +167,8 @@ class ImageUploader(object):
pass
@abc.abstractmethod
def discover_image_tag(self, image, tag_from_label=None):
def discover_image_tag(self, image, tag_from_label=None,
username=None, password=None):
"""Discover a versioned tag for an image"""
pass
@ -267,10 +273,10 @@ class BaseImageUploader(ImageUploader):
'Modifying image %s failed' % target_image)
@staticmethod
def _images_match(image1, image2, insecure_registries):
def _images_match(image1, image2, insecure_registries, session1=None):
try:
image1_digest = BaseImageUploader._image_digest(
image1, insecure_registries)
image1, insecure_registries, session=session1)
except Exception:
return False
try:
@ -285,27 +291,74 @@ class BaseImageUploader(ImageUploader):
return image1_digest == image2_digest
@staticmethod
def _image_digest(image, insecure_registries):
def _image_digest(image, insecure_registries, session=None):
image_url = BaseImageUploader._image_to_url(image)
insecure = image_url.netloc in insecure_registries
i = BaseImageUploader._inspect(image_url, insecure)
i = BaseImageUploader._inspect(image_url, insecure, session)
return i.get('Digest')
@staticmethod
def _image_labels(image, insecure):
image_url = BaseImageUploader._image_to_url(image)
i = BaseImageUploader._inspect(image_url, insecure)
def _image_labels(image_url, insecure, session=None):
i = BaseImageUploader._inspect(image_url, insecure, session)
return i.get('Labels', {}) or {}
@staticmethod
def _image_exists(image, insecure_registries):
def _image_exists(image, insecure_registries, session=None):
try:
BaseImageUploader._image_digest(image, insecure_registries)
BaseImageUploader._image_digest(
image, insecure_registries, session=session)
except ImageNotFoundException:
return False
else:
return True
@staticmethod
def authenticate(image_url, username=None, password=None, insecure=False):
image_url = BaseImageUploader._fix_dockerio_url(image_url)
netloc = image_url.netloc
if insecure:
scheme = 'http'
else:
scheme = 'https'
image, tag = image_url.path.split(':')
url = '%s://%s/v2/' % (scheme, netloc)
session = requests.Session()
r = session.get(url, timeout=30)
LOG.debug('%s status code %s' % (url, r.status_code))
if r.status_code != 401:
return session
if 'www-authenticate' not in r.headers:
raise ImageUploaderException(
'Unknown authentication method for headers: %s' % r.headers)
www_auth = r.headers['www-authenticate']
if not www_auth.startswith('Bearer '):
raise ImageUploaderException(
'Unknown www-authenticate value: %s' % www_auth)
token_param = {}
realm = re.search('realm="(.*?)"', www_auth).group(1)
token_param['service'] = re.search(
'service="(.*?)"', www_auth).group(1)
token_param['scope'] = 'repository:%s:pull' % image[1:]
auth = None
if username:
auth = requests_auth.HTTPBasicAuth(username, password)
rauth = session.get(realm, params=token_param, auth=auth, timeout=30)
rauth.raise_for_status()
session.headers['Authorization'] = 'Bearer %s' % rauth.json()['token']
return session
@staticmethod
def _fix_dockerio_url(url):
one = 'docker.io'
two = 'registry-1.docker.io'
if url.netloc != one:
return url
return parse.ParseResult(url.scheme, two,
url.path, url.params,
url.query, url.fragment)
@staticmethod
@tenacity.retry( # Retry up to 5 times with jittered exponential backoff
reraise=True,
@ -313,39 +366,79 @@ class BaseImageUploader(ImageUploader):
wait=tenacity.wait_random_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(5)
)
def _inspect(image_url, insecure=False):
image = image_url.geturl()
cmd = ['skopeo', 'inspect']
def _inspect(image_url, insecure=False, session=None):
original_image_url = image_url
image_url = BaseImageUploader._fix_dockerio_url(image_url)
parts = {
'netloc': image_url.netloc
}
if insecure:
cmd.append('--tls-verify=false')
cmd.append(image)
parts['scheme'] = 'http'
else:
parts['scheme'] = 'https'
image, tag = image_url.path.split(':')
parts['image'] = image
parts['tag'] = tag
LOG.info('Running %s' % ' '.join(cmd))
env = os.environ.copy()
process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
manifest_url = ('%(scheme)s://%(netloc)s/v2'
'%(image)s/manifests/%(tag)s' % parts)
tags_url = ('%(scheme)s://%(netloc)s/v2'
'%(image)s/tags/list' % parts)
manifest_headers = {
'Accept': 'application/vnd.docker.distribution.manifest.v2+json'
}
out, err = process.communicate()
if process.returncode != 0:
not_found_msgs = (
u'manifest unknown',
# returned by docker.io
u'requested access to the resource is denied'
)
if any(n in err for n in not_found_msgs):
raise ImageNotFoundException('Not found image: %s\n%s' %
(image, err))
raise ImageUploaderException('Error inspecting image: %s\n%s' %
(image, err))
return json.loads(out)
p = futures.ThreadPoolExecutor(max_workers=2)
manifest_f = p.submit(
session.get, manifest_url, headers=manifest_headers, timeout=30)
tags_f = p.submit(session.get, tags_url, timeout=30)
manifest_r = manifest_f.result()
tags_r = tags_f.result()
if manifest_r.status_code == 404:
raise ImageNotFoundException('Not found image: %s' %
image_url.geturl())
manifest_r.raise_for_status()
tags_r.raise_for_status()
manifest = manifest_r.json()
layers = [l['digest'] for l in manifest['layers']]
parts['config_digest'] = manifest['config']['digest']
config_headers = {
'Accept': manifest['config']['mediaType']
}
config_url = ('%(scheme)s://%(netloc)s/v2'
'%(image)s/blobs/%(config_digest)s' % parts)
config_f = p.submit(
session.get, config_url, headers=config_headers, timeout=30)
config_r = config_f.result()
config_r.raise_for_status()
tags = tags_r.json()['tags']
digest = manifest_r.headers['Docker-Content-Digest']
config = config_r.json()
name = '%s%s' % (original_image_url.netloc, image)
return {
'Name': name,
'Digest': digest,
'RepoTags': tags,
'Created': config['created'],
'DockerVersion': config['docker_version'],
'Labels': config['config']['Labels'],
'Architecture': config['architecture'],
'Os': config['os'],
'Layers': layers,
}
@staticmethod
def _image_to_url(image):
if '://' not in image:
image = 'docker://' + image
return urlparse(image)
url = parse.urlparse(image)
return url
@staticmethod
def _discover_tag_from_inspect(i, image, tag_from_label=None,
@ -412,20 +505,25 @@ class BaseImageUploader(ImageUploader):
return versioned_images
def discover_image_tag(self, image, tag_from_label=None,
fallback_tag=None):
fallback_tag=None, username=None, password=None):
image_url = self._image_to_url(image)
insecure = self.is_insecure_registry(image_url.netloc)
i = self._inspect(image_url, insecure)
session = self.authenticate(
image_url, insecure=insecure, username=username, password=password)
i = self._inspect(image_url, insecure, session)
return self._discover_tag_from_inspect(i, image, tag_from_label,
fallback_tag)
def filter_images_with_labels(self, images, labels):
def filter_images_with_labels(self, images, labels,
username=None, password=None):
images_with_labels = []
for image in images:
url = self._image_to_url(image)
image_labels = self._image_labels(url.geturl(),
self.is_insecure_registry(
url.netloc))
insecure = self.is_insecure_registry(url.netloc)
session = self.authenticate(
url, insecure=insecure, username=username, password=password)
image_labels = self._image_labels(
url, insecure=insecure, session=session)
if set(labels).issubset(set(image_labels)):
images_with_labels.append(image)
@ -465,7 +563,7 @@ class BaseImageUploader(ImageUploader):
@staticmethod
def _cross_repo_mount(target_image_url, image_layers,
source_layers, insecure_registries):
source_layers, insecure_registries, session):
netloc = target_image_url.netloc
name = target_image_url.path.split(':')[0][1:]
if netloc in insecure_registries:
@ -483,7 +581,7 @@ class BaseImageUploader(ImageUploader):
'mount': layer,
'from': existing_name
}
r = requests.post(url, data=data)
r = session.post(url, data=data)
LOG.debug('%s %s' % (r.status_code, r.reason))
@ -500,24 +598,34 @@ class DockerImageUploader(BaseImageUploader):
source_tag = names['source_tag']
repo = names['repo']
source_image = names['source_image']
source_image_url = BaseImageUploader._image_to_url(source_image)
source_insecure = source_image_url.netloc in insecure_registries
target_image_no_tag = names['target_image_no_tag']
append_tag = names['append_tag']
target_tag = names['target_tag']
target_image_source_tag = names['target_image_source_tag']
target_image = names['target_image']
target_image_url = BaseImageUploader._image_to_url(target_image)
target_insecure = target_image_url.netloc in insecure_registries
if dry_run:
return []
if modify_role:
target_session = BaseImageUploader.authenticate(
target_image_url, insecure=target_insecure)
if BaseImageUploader._image_exists(target_image,
insecure_registries):
insecure_registries,
session=target_session):
LOG.warning('Skipping upload for modified image %s' %
target_image)
return []
else:
source_session = BaseImageUploader.authenticate(
source_image_url, insecure=source_insecure)
if BaseImageUploader._images_match(source_image, target_image,
insecure_registries):
insecure_registries,
session1=source_session):
LOG.warning('Skipping upload for image %s' % image_name)
return []
@ -631,29 +739,40 @@ class SkopeoImageUploader(BaseImageUploader):
source_image = names['source_image']
source_image_url = BaseImageUploader._image_to_url(source_image)
source_image_local_url = urlparse('containers-storage:%s'
% source_image)
source_image_local_url = parse.urlparse('containers-storage:%s'
% source_image)
source_insecure = source_image_url.netloc in insecure_registries
append_tag = names['append_tag']
target_image_source_tag = names['target_image_source_tag']
target_image = names['target_image']
target_image_url = BaseImageUploader._image_to_url(target_image)
target_image_local_url = urlparse('containers-storage:%s' %
target_image)
target_image_local_url = parse.urlparse('containers-storage:%s' %
target_image)
target_insecure = target_image_local_url.netloc in insecure_registries
if dry_run:
return []
target_session = BaseImageUploader.authenticate(
target_image_url, insecure=target_insecure)
if modify_role and BaseImageUploader._image_exists(
target_image, insecure_registries):
target_image, insecure_registries, target_session):
LOG.warning('Skipping upload for modified image %s' %
target_image)
return []
source_inspect = BaseImageUploader._inspect(source_image_url)
source_session = BaseImageUploader.authenticate(
source_image_url, insecure=source_insecure)
source_inspect = BaseImageUploader._inspect(
source_image_url, insecure=source_insecure, session=source_session)
source_layers = source_inspect.get('Layers', [])
BaseImageUploader._cross_repo_mount(
target_image_url, image_layers, source_layers, insecure_registries)
target_image_url, image_layers, source_layers, insecure_registries,
session=source_session)
to_cleanup = []
if modify_role:
@ -671,8 +790,6 @@ class SkopeoImageUploader(BaseImageUploader):
modify_role, modify_vars, source_image,
target_image_source_tag, append_tag,
container_build_tool='buildah')
# Inspect to confirm the playbook created the target image
BaseImageUploader._inspect(target_image_local_url)
if cleanup == CLEANUP_FULL:
to_cleanup.append(target_image)
@ -756,7 +873,7 @@ class SkopeoImageUploader(BaseImageUploader):
if not image:
continue
LOG.warning('Removing local copy of %s' % image)
image_url = urlparse('containers-storage:%s' % image)
image_url = parse.urlparse('containers-storage:%s' % image)
SkopeoImageUploader._delete(image_url)
def run_tasks(self):
@ -795,7 +912,9 @@ def discover_tag_from_inspect(args):
image, tag_from_label, insecure_registries = args
image_url = BaseImageUploader._image_to_url(image)
insecure = image_url.netloc in insecure_registries
i = BaseImageUploader._inspect(image_url, insecure)
session = BaseImageUploader.authenticate(image_url, insecure=insecure)
i = BaseImageUploader._inspect(image_url, insecure=insecure,
session=session)
if ':' in image_url.path:
# break out the tag from the url to be the fallback tag
path = image.rpartition(':')

313
tripleo_common/tests/image/test_image_uploader.py
View File

@ -13,11 +13,11 @@
# under the License.
#
import json
import mock
import operator
import os
import requests
from requests_mock.contrib import fixture as rm_fixture
import six
from six.moves.urllib.parse import urlparse
import tempfile
@ -50,6 +50,8 @@ class TestImageUploadManager(base.TestCase):
files.append('testfile')
self.filelist = files
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.image.base.open',
@ -66,7 +68,8 @@ class TestImageUploadManager(base.TestCase):
@mock.patch('tripleo_common.image.image_uploader.'
'get_undercloud_registry', return_value='192.0.2.0:8787')
def test_file_parsing(self, mock_gur, mockdocker, mockioctl, mockpath,
mock_images_match, mock_is_insecure, mock_inspect):
mock_images_match, mock_is_insecure, mock_inspect,
mock_auth):
mock_inspect.return_value = {}
manager = image_uploader.ImageUploadManager(self.filelist, debug=True)
@ -171,89 +174,101 @@ class TestBaseImageUploader(base.TestCase):
super(TestBaseImageUploader, self).setUp()
self.uploader = image_uploader.BaseImageUploader()
self.uploader._inspect.retry.sleep = mock.Mock()
self.requests = self.useFixture(rm_fixture.Fixture())
@mock.patch('requests.get')
def test_is_insecure_registry_known(self, mock_get):
def test_is_insecure_registry_known(self):
self.assertFalse(
self.uploader.is_insecure_registry('docker.io'))
@mock.patch('requests.get')
def test_is_insecure_registry_secure(self, mock_get):
def test_is_insecure_registry_secure(self):
self.assertFalse(
self.uploader.is_insecure_registry('192.0.2.0:8787'))
self.assertFalse(
self.uploader.is_insecure_registry('192.0.2.0:8787'))
mock_get.assert_called_once_with('https://192.0.2.0:8787/')
self.assertEqual(
'https://192.0.2.0:8787/',
self.requests.request_history[0].url
)
@mock.patch('requests.get')
def test_is_insecure_registry_timeout(self, mock_get):
mock_get.side_effect = requests.exceptions.ReadTimeout('ouch')
def test_is_insecure_registry_timeout(self):
self.requests.get(
'https://192.0.2.0:8787/',
exc=requests.exceptions.ReadTimeout('ouch'))
self.assertFalse(
self.uploader.is_insecure_registry('192.0.2.0:8787'))
self.assertFalse(
self.uploader.is_insecure_registry('192.0.2.0:8787'))
mock_get.assert_called_once_with('https://192.0.2.0:8787/')
self.assertEqual(
'https://192.0.2.0:8787/',
self.requests.request_history[0].url
)
@mock.patch('requests.get')
def test_is_insecure_registry_insecure(self, mock_get):
mock_get.side_effect = requests.exceptions.SSLError('ouch')
def test_is_insecure_registry_insecure(self):
self.requests.get(
'https://192.0.2.0:8787/',
exc=requests.exceptions.SSLError('ouch'))
self.assertTrue(
self.uploader.is_insecure_registry('192.0.2.0:8787'))
self.assertTrue(
self.uploader.is_insecure_registry('192.0.2.0:8787'))
mock_get.assert_called_once_with('https://192.0.2.0:8787/')
self.assertEqual(
'https://192.0.2.0:8787/',
self.requests.request_history[0].url
)
@mock.patch('subprocess.Popen')
def test_discover_image_tag(self, mock_popen):
result = {
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
def test_discover_image_tag(self, mock_inspect, mock_auth):
mock_inspect.return_value = {
'Labels': {
'rdo_version': 'a',
'build_version': '4.0.0'
},
'RepoTags': ['a']
}
mock_process = mock.Mock()
mock_process.communicate.return_value = (json.dumps(result), '')
mock_process.returncode = 0
mock_popen.return_value = mock_process
self.assertEqual(
'a',
self.uploader.discover_image_tag('docker.io/t/foo', 'rdo_version')
self.uploader.discover_image_tag('docker.io/t/foo:b',
'rdo_version')
)
# no tag_from_label specified
self.assertRaises(
ImageUploaderException,
self.uploader.discover_image_tag,
'docker.io/t/foo')
'docker.io/t/foo:b')
# missing RepoTags entry
self.assertRaises(
ImageUploaderException,
self.uploader.discover_image_tag,
'docker.io/t/foo',
'docker.io/t/foo:b',
'build_version')
# missing Labels entry
self.assertRaises(
ImageUploaderException,
self.uploader.discover_image_tag,
'docker.io/t/foo',
'docker.io/t/foo:b',
'version')
# inspect call failed
mock_process.returncode = 1
mock_process.communicate.return_value = ('', 'manifest unknown')
mock_inspect.side_effect = ImageNotFoundException()
self.assertRaises(
ImageNotFoundException,
self.uploader.discover_image_tag,
'docker.io/t/foo',
'docker.io/t/foo:b',
'rdo_version')
@mock.patch('subprocess.Popen')
def test_discover_tag_from_inspect(self, mock_popen):
result = {
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
def test_discover_tag_from_inspect(self, mock_inspect, mock_auth):
mock_inspect.return_value = {
'Labels': {
'rdo_version': 'a',
'build_version': '4.0.0',
@ -262,10 +277,6 @@ class TestBaseImageUploader(base.TestCase):
},
'RepoTags': ['a', '1.0.0-20180125']
}
mock_process = mock.Mock()
mock_process.communicate.return_value = (json.dumps(result), '')
mock_process.returncode = 0
mock_popen.return_value = mock_process
sr = image_uploader.SECURE_REGISTRIES
# simple label -> tag
@ -332,7 +343,7 @@ class TestBaseImageUploader(base.TestCase):
)
# inspect call failed
mock_process.returncode = 1
mock_inspect.side_effect = ImageUploaderException()
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
@ -388,6 +399,119 @@ class TestBaseImageUploader(base.TestCase):
mock_inspect.side_effect = ImageUploaderException()
self.assertFalse(self.uploader._images_match('foo', 'bar', set()))
def test_authenticate(self):
req = self.requests
auth = image_uploader.BaseImageUploader.authenticate
url1 = urlparse('docker://docker.io/t/nova-api:latest')
# no auth required
req.get('https://registry-1.docker.io/v2/', status_code=200)
self.assertNotIn('Authorization', auth(url1).headers)
# missing 'www-authenticate' header
req.get('https://registry-1.docker.io/v2/', status_code=401)
self.assertRaises(ImageUploaderException, auth, url1)
# unknown 'www-authenticate' header
req.get('https://registry-1.docker.io/v2/', status_code=401,
headers={'www-authenticate': 'Foo'})
self.assertRaises(ImageUploaderException, auth, url1)
# successful auth requests
headers = {
'www-authenticate': 'Bearer '
'realm="https://auth.docker.io/token",'
'service="registry.docker.io"'
}
req.get('https://registry-1.docker.io/v2/', status_code=401,
headers=headers)
req.get('https://auth.docker.io/token', json={"token": "asdf1234"})
self.assertEqual(
'Bearer asdf1234',
auth(url1).headers['Authorization']
)
def test_fix_dockerio_url(self):
url1 = urlparse('docker://docker.io/t/nova-api:latest')
url2 = urlparse('docker://registry-1.docker.io/t/nova-api:latest')
url3 = urlparse('docker://192.0.2.1:8787/t/nova-api:latest')
fix = image_uploader.BaseImageUploader._fix_dockerio_url
# fix urls
self.assertEqual(url2, fix(url1))
# no change urls
self.assertEqual(url2, fix(url2))
self.assertEqual(url3, fix(url3))
def test_inspect(self):
req = self.requests
session = requests.Session()
session.headers['Authorization'] = 'Bearer asdf1234'
inspect = image_uploader.BaseImageUploader._inspect
url1 = urlparse('docker://docker.io/t/nova-api:latest')
manifest_resp = {
'config': {
'mediaType': 'text/html',
'digest': 'abcdef'
},
'layers': [
{'digest': 'aaa'},
{'digest': 'bbb'},
{'digest': 'ccc'},
]
}
manifest_headers = {'Docker-Content-Digest': 'eeeeee'}
tags_resp = {'tags': ['one', 'two', 'latest']}
config_resp = {
'created': '2018-10-02T11:13:45.567533229Z',
'docker_version': '1.13.1',
'config': {
'Labels': {
'build-date': '20181002',
'build_id': '1538477701',
'kolla_version': '7.0.0'
}
},
'architecture': 'amd64',
'os': 'linux',
}
req.get('https://registry-1.docker.io/v2/t/nova-api/tags/list',
json=tags_resp)
req.get('https://registry-1.docker.io/v2/t/nova-api/blobs/abcdef',
json=config_resp)
# test 404 response
req.get('https://registry-1.docker.io/v2/t/nova-api/manifests/latest',
status_code=404)
self.assertRaises(ImageNotFoundException, inspect, url1,
session=session)
# test full response
req.get('https://registry-1.docker.io/v2/t/nova-api/manifests/latest',
json=manifest_resp, headers=manifest_headers)
self.assertEqual(
{
'Architecture': 'amd64',
'Created': '2018-10-02T11:13:45.567533229Z',
'Digest': 'eeeeee',
'DockerVersion': '1.13.1',
'Labels': {
'build-date': '20181002',
'build_id': '1538477701',
'kolla_version': '7.0.0'
},
'Layers': ['aaa', 'bbb', 'ccc'],
'Name': 'docker.io/t/nova-api',
'Os': 'linux',
'RepoTags': ['one', 'two', 'latest']
},
inspect(url1, session=session)
)
class TestDockerImageUploader(base.TestCase):
@ -405,22 +529,18 @@ class TestDockerImageUploader(base.TestCase):
super(TestDockerImageUploader, self).tearDown()
self.patcher.stop()
@mock.patch('subprocess.Popen')
def test_upload_image(self, mock_popen):
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
def test_upload_image(self, mock_inspect, mock_auth):
result1 = {
'Digest': 'a'
}
result2 = {
'Digest': 'b'
}
mock_process = mock.Mock()
mock_process.communicate.side_effect = [
(json.dumps(result1), ''),
(json.dumps(result2), ''),
]
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_inspect.side_effect = [result1, result2]
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
tag = 'latest'
@ -457,46 +577,14 @@ class TestDockerImageUploader(base.TestCase):
push_image,
tag=tag, stream=True)
@mock.patch('subprocess.Popen')
def test_upload_image_missing_tag(self, mock_popen):
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
expected_tag = 'latest'
push_destination = 'localhost:8787'
push_image = 'localhost:8787/tripleomaster/heat-docker-agents-centos'
self.uploader.upload_image(image,
None,
push_destination,
set(),
None,
None,
None,
False,
'full',
{})
self.dockermock.assert_called_once_with(
base_url='unix://var/run/docker.sock', version='auto')
self.dockermock.return_value.pull.assert_called_once_with(
image, tag=expected_tag, stream=True)
self.dockermock.return_value.tag.assert_called_once_with(
image=image + ':' + expected_tag,
repository=push_image,
tag=expected_tag, force=True)
self.dockermock.return_value.push.assert_called_once_with(
push_image,
tag=expected_tag, stream=True)
@mock.patch('subprocess.Popen')
def test_upload_image_existing(self, mock_popen):
result = {
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
def test_upload_image_existing(self, mock_inspect, mock_auth):
mock_inspect.return_value = {
'Digest': 'a'
}
mock_process = mock.Mock()
mock_process.communicate.return_value = (json.dumps(result), '')
mock_process.returncode = 0
mock_popen.return_value = mock_process
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
tag = 'latest'
push_destination = 'localhost:8787'
@ -523,16 +611,14 @@ class TestDockerImageUploader(base.TestCase):
self.dockermock.return_value.tag.assert_not_called()
self.dockermock.return_value.push.assert_not_called()
@mock.patch('subprocess.Popen')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_upload_image(self, mock_ansible, mock_popen):
mock_process = mock.Mock()
mock_process.communicate.return_value = (
'', 'FATA[0000] Error reading manifest: manifest unknown')
mock_process.returncode = 1
mock_popen.return_value = mock_process
def test_modify_upload_image(self, mock_ansible, mock_inspect, mock_auth):
mock_inspect.side_effect = ImageNotFoundException()
mock_ansible.return_value.run.return_value = {}
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
@ -591,15 +677,14 @@ class TestDockerImageUploader(base.TestCase):
tag=tag + append_tag,
stream=True)
@mock.patch('subprocess.Popen')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_image_failed(self, mock_ansible, mock_popen):
mock_process = mock.Mock()
mock_process.communicate.return_value = ('', 'manifest unknown')
mock_process.returncode = 1
mock_popen.return_value = mock_process
def test_modify_image_failed(self, mock_ansible, mock_inspect, mock_auth):
mock_inspect.side_effect = ImageNotFoundException()
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
tag = 'latest'
@ -655,11 +740,14 @@ class TestDockerImageUploader(base.TestCase):
mock_process.communicate.assert_not_called()
self.assertEqual([], result)
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_image_existing(self, mock_ansible, mock_inspect):
def test_modify_image_existing(self, mock_ansible, mock_inspect,
mock_auth):
mock_inspect.return_value = {'Digest': 'a'}
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
@ -772,7 +860,10 @@ class TestSkopeoImageUploader(base.TestCase):
@mock.patch('subprocess.Popen')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
def test_upload_image(self, mock_inspect, mock_popen, mock_environ):
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
def test_upload_image(self, mock_auth, mock_inspect,
mock_popen, mock_environ):
mock_process = mock.Mock()
mock_process.communicate.return_value = ('copy complete', '')
mock_process.returncode = 0
@ -807,6 +898,8 @@ class TestSkopeoImageUploader(base.TestCase):
env={}, stdout=-1
)
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.image.image_uploader.'
@ -816,7 +909,7 @@ class TestSkopeoImageUploader(base.TestCase):
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_upload_image(self, mock_ansible, mock_exists, mock_copy,
mock_inspect):
mock_inspect, mock_auth):
mock_exists.return_value = False
mock_inspect.return_value = {}
with tempfile.NamedTemporaryFile(delete=False) as logfile:
@ -868,10 +961,7 @@ class TestSkopeoImageUploader(base.TestCase):
mock_inspect.assert_has_calls([
mock.call(urlparse(
'docker://docker.io/t/nova-api:latest'
)),
mock.call(urlparse(
'containers-storage:localhost:8787/t/nova-api:latestmodify-123'
))
), insecure=False, session=mock.ANY)
])
mock_copy.assert_has_calls([
mock.call(
@ -894,6 +984,8 @@ class TestSkopeoImageUploader(base.TestCase):
extra_env_variables=mock.ANY
)
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.image.image_uploader.'
@ -903,7 +995,7 @@ class TestSkopeoImageUploader(base.TestCase):
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_image_failed(self, mock_ansible, mock_exists, mock_copy,
mock_inspect):
mock_inspect, mock_auth):
mock_exists.return_value = False
mock_inspect.return_value = {}
@ -959,11 +1051,14 @@ class TestSkopeoImageUploader(base.TestCase):
mock_process.communicate.assert_not_called()
self.assertEqual([], result)
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_image_existing(self, mock_ansible, mock_inspect):
def test_modify_image_existing(self, mock_ansible, mock_inspect,
mock_auth):
mock_inspect.return_value = {'Digest': 'a'}
image = 'docker.io/t/nova-api'