Allow PythonImageUploader to accept unknown CA
Handles unknown CA in a dedicated list in order to know when to enable "verify" for requests.Session calls. This new list will hold the registries with unknown CA like it's done for the "insecure registries" (this one means "no encryption" aka "http"). Change-Id: I00b2e59d3da5374f20dc2eac9bb13e2482ed524b Related-Bug: #1817360
This commit is contained in:
Cédric Jeanneret
parent
6b10061115
commit
6110802999
|
|
@ -0,0 +1,3 @@
|
|||
---
|
||||
features:
|
||||
- add support for unknown CA
|
||||
|
|
@ -48,6 +48,8 @@ SECURE_REGISTRIES = (
|
|||
'registry-1.docker.io',
|
||||
)
|
||||
|
||||
NO_VERIFY_REGISTRIES = ()
|
||||
|
||||
CLEANUP = (
|
||||
CLEANUP_FULL, CLEANUP_PARTIAL, CLEANUP_NONE
|
||||
) = (
|
||||
|
|
@ -183,6 +185,7 @@ class BaseImageUploader(object):
|
|||
|
||||
mirrors = {}
|
||||
insecure_registries = set()
|
||||
no_verify_registries = set(NO_VERIFY_REGISTRIES)
|
||||
secure_registries = set(SECURE_REGISTRIES)
|
||||
export_registries = set()
|
||||
push_registries = set()
|
||||
|
|
@ -196,6 +199,8 @@ class BaseImageUploader(object):
|
|||
@classmethod
|
||||
def init_registries_cache(cls):
|
||||
cls.insecure_registries.clear()
|
||||
cls.no_verify_registries.clear()
|
||||
cls.no_verify_registries.update(NO_VERIFY_REGISTRIES)
|
||||
cls.secure_registries.clear()
|
||||
cls.secure_registries.update(SECURE_REGISTRIES)
|
||||
cls.mirrors.clear()
|
||||
|
|
@ -294,7 +299,6 @@ class BaseImageUploader(object):
|
|||
else:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
@tenacity.retry( # Retry up to 5 times with jittered exponential backoff
|
||||
reraise=True,
|
||||
retry=tenacity.retry_if_exception_type(
|
||||
|
|
@ -303,10 +307,13 @@ class BaseImageUploader(object):
|
|||
wait=tenacity.wait_random_exponential(multiplier=1, max=10),
|
||||
stop=tenacity.stop_after_attempt(5)
|
||||
)
|
||||
def authenticate(cls, image_url, username=None, password=None):
|
||||
image, tag = cls._image_tag_from_url(image_url)
|
||||
url = cls._build_url(image_url, path='/')
|
||||
def authenticate(self, image_url, username=None, password=None):
|
||||
netloc = image_url.netloc
|
||||
image, tag = self._image_tag_from_url(image_url)
|
||||
url = self._build_url(image_url, path='/')
|
||||
|
||||
session = requests.Session()
|
||||
session.verify = (netloc not in self.no_verify_registries)
|
||||
r = session.get(url, timeout=30)
|
||||
LOG.debug('%s status code %s' % (url, r.status_code))
|
||||
if r.status_code == 200:
|
||||
|
|
@ -340,12 +347,16 @@ class BaseImageUploader(object):
|
|||
def _build_url(cls, url, path):
|
||||
netloc = url.netloc
|
||||
insecure = netloc in cls.insecure_registries
|
||||
tls_verify = netloc in cls.no_verify_registries
|
||||
if netloc in cls.mirrors:
|
||||
mirror = cls.mirrors[netloc]
|
||||
return '%sv2%s' % (mirror, path)
|
||||
else:
|
||||
if insecure:
|
||||
scheme = 'http'
|
||||
# Just to be clear: we DO want TLS for that specific case
|
||||
elif tls_verify:
|
||||
scheme = 'https'
|
||||
else:
|
||||
scheme = 'https'
|
||||
if netloc == 'docker.io':
|
||||
|
|
@ -398,6 +409,7 @@ class BaseImageUploader(object):
|
|||
tags_r.raise_for_status()
|
||||
|
||||
manifest = manifest_r.json()
|
||||
|
||||
digest = manifest_r.headers['Docker-Content-Digest']
|
||||
if manifest.get('schemaVersion', 2) == 1:
|
||||
config = json.loads(manifest['history'][0]['v1Compatibility'])
|
||||
|
|
@ -498,11 +510,11 @@ class BaseImageUploader(object):
|
|||
|
||||
# prime self.insecure_registries by testing every image
|
||||
for url in image_urls:
|
||||
self.is_insecure_registry(url.netloc)
|
||||
self.is_insecure_registry(url)
|
||||
|
||||
discover_args = []
|
||||
for image in images:
|
||||
discover_args.append((image, tag_from_label))
|
||||
discover_args.append((self, image, tag_from_label))
|
||||
p = futures.ThreadPoolExecutor(max_workers=16)
|
||||
|
||||
versioned_images = {}
|
||||
|
|
@ -517,6 +529,7 @@ class BaseImageUploader(object):
|
|||
self.is_insecure_registry(image_url.netloc)
|
||||
session = self.authenticate(
|
||||
image_url, username=username, password=password)
|
||||
|
||||
i = self._inspect(image_url, session)
|
||||
return self._discover_tag_from_inspect(i, image, tag_from_label,
|
||||
fallback_tag)
|
||||
|
|
@ -553,11 +566,22 @@ class BaseImageUploader(object):
|
|||
return False
|
||||
if registry_host in self.insecure_registries:
|
||||
return True
|
||||
if registry_host in self.no_verify_registries:
|
||||
return True
|
||||
try:
|
||||
requests.get('https://%s/v2' % registry_host, timeout=30)
|
||||
except requests.exceptions.SSLError:
|
||||
self.insecure_registries.add(registry_host)
|
||||
return True
|
||||
# Might be just a TLS certificate validation issue
|
||||
# Just retry without the verification
|
||||
try:
|
||||
requests.get('https://%s/v2' % registry_host, timeout=30,
|
||||
verify=False)
|
||||
self.no_verify_registries.add(registry_host)
|
||||
return True
|
||||
except requests.exceptions.SSLError:
|
||||
# So nope, it's really not a certificate verification issue
|
||||
self.insecure_registries.add(registry_host)
|
||||
return True
|
||||
except Exception:
|
||||
# for any other error assume it is a secure registry, because:
|
||||
# - it is secure registry
|
||||
|
|
@ -780,7 +804,8 @@ class PythonImageUploader(BaseImageUploader):
|
|||
return []
|
||||
|
||||
target_session = self.authenticate(
|
||||
t.target_image_url)
|
||||
t.target_image_url
|
||||
)
|
||||
|
||||
self._detect_target_export(t.target_image_url, target_session)
|
||||
|
||||
|
|
@ -795,7 +820,8 @@ class PythonImageUploader(BaseImageUploader):
|
|||
copy_target_url = t.target_image_url
|
||||
|
||||
source_session = self.authenticate(
|
||||
t.source_image_url)
|
||||
t.source_image_url
|
||||
)
|
||||
|
||||
manifest_str = self._fetch_manifest(
|
||||
t.source_image_url,
|
||||
|
|
@ -1147,7 +1173,8 @@ class PythonImageUploader(BaseImageUploader):
|
|||
LOG.info('Pulling %s' % pull_source)
|
||||
cmd = ['podman', 'pull']
|
||||
|
||||
if source_url.netloc in cls.insecure_registries:
|
||||
if source_url.netloc in [cls.insecure_registries,
|
||||
cls.no_verify_registries]:
|
||||
cmd.append('--tls-verify=false')
|
||||
|
||||
cmd.append(pull_source)
|
||||
|
|
@ -1536,9 +1563,9 @@ def upload_task(args):
|
|||
|
||||
|
||||
def discover_tag_from_inspect(args):
|
||||
image, tag_from_label = args
|
||||
self, image, tag_from_label = args
|
||||
image_url = BaseImageUploader._image_to_url(image)
|
||||
session = BaseImageUploader.authenticate(image_url)
|
||||
session = self.authenticate(image_url)
|
||||
i = BaseImageUploader._inspect(image_url, session=session)
|
||||
if ':' in image_url.path:
|
||||
# break out the tag from the url to be the fallback tag
|
||||
|
|
|
|||
|
|
@ -270,63 +270,63 @@ class TestBaseImageUploader(base.TestCase):
|
|||
self.assertEqual(
|
||||
('docker.io/t/foo', 'a'),
|
||||
image_uploader.discover_tag_from_inspect(
|
||||
('docker.io/t/foo', 'rdo_version'))
|
||||
(self.uploader, 'docker.io/t/foo', 'rdo_version'))
|
||||
)
|
||||
|
||||
# templated labels -> tag
|
||||
self.assertEqual(
|
||||
('docker.io/t/foo', '1.0.0-20180125'),
|
||||
image_uploader.discover_tag_from_inspect(
|
||||
('docker.io/t/foo', '{release}-{version}'))
|
||||
(self.uploader, 'docker.io/t/foo', '{release}-{version}'))
|
||||
)
|
||||
|
||||
# simple label -> tag with fallback
|
||||
self.assertEqual(
|
||||
('docker.io/t/foo', 'a'),
|
||||
image_uploader.discover_tag_from_inspect(
|
||||
('docker.io/t/foo:a', 'bar'))
|
||||
(self.uploader, 'docker.io/t/foo:a', 'bar'))
|
||||
)
|
||||
|
||||
# templated labels -> tag with fallback
|
||||
self.assertEqual(
|
||||
('docker.io/t/foo', 'a'),
|
||||
image_uploader.discover_tag_from_inspect(
|
||||
('docker.io/t/foo:a', '{releases}-{versions}'))
|
||||
(self.uploader, 'docker.io/t/foo:a', '{releases}-{versions}'))
|
||||
)
|
||||
|
||||
# Invalid template
|
||||
self.assertRaises(
|
||||
ImageUploaderException,
|
||||
image_uploader.discover_tag_from_inspect,
|
||||
('docker.io/t/foo', '{release}-{version')
|
||||
(self.uploader, 'docker.io/t/foo', '{release}-{version')
|
||||
)
|
||||
|
||||
# Missing label in template
|
||||
self.assertRaises(
|
||||
ImageUploaderException,
|
||||
image_uploader.discover_tag_from_inspect,
|
||||
('docker.io/t/foo', '{releases}-{version}')
|
||||
(self.uploader, 'docker.io/t/foo', '{releases}-{version}')
|
||||
)
|
||||
|
||||
# no tag_from_label specified
|
||||
self.assertRaises(
|
||||
ImageUploaderException,
|
||||
image_uploader.discover_tag_from_inspect,
|
||||
('docker.io/t/foo', None)
|
||||
(self.uploader, 'docker.io/t/foo', None)
|
||||
)
|
||||
|
||||
# missing RepoTags entry
|
||||
self.assertRaises(
|
||||
ImageUploaderException,
|
||||
image_uploader.discover_tag_from_inspect,
|
||||
('docker.io/t/foo', 'build_version')
|
||||
(self.uploader, 'docker.io/t/foo', 'build_version')
|
||||
)
|
||||
|
||||
# missing Labels entry
|
||||
self.assertRaises(
|
||||
ImageUploaderException,
|
||||
image_uploader.discover_tag_from_inspect,
|
||||
('docker.io/t/foo', 'version')
|
||||
(self.uploader, 'docker.io/t/foo', 'version')
|
||||
)
|
||||
|
||||
# inspect call failed
|
||||
|
|
@ -334,7 +334,7 @@ class TestBaseImageUploader(base.TestCase):
|
|||
self.assertRaises(
|
||||
ImageUploaderException,
|
||||
image_uploader.discover_tag_from_inspect,
|
||||
('docker.io/t/foo', 'rdo_version')
|
||||
(self.uploader, 'docker.io/t/foo', 'rdo_version')
|
||||
)
|
||||
|
||||
@mock.patch('concurrent.futures.ThreadPoolExecutor')
|
||||
|
|
@ -360,9 +360,9 @@ class TestBaseImageUploader(base.TestCase):
|
|||
mock_pool.return_value.map.assert_called_once_with(
|
||||
image_uploader.discover_tag_from_inspect,
|
||||
[
|
||||
('docker.io/t/foo', 'rdo_release'),
|
||||
('docker.io/t/bar', 'rdo_release'),
|
||||
('docker.io/t/baz', 'rdo_release')
|
||||
(self.uploader, 'docker.io/t/foo', 'rdo_release'),
|
||||
(self.uploader, 'docker.io/t/bar', 'rdo_release'),
|
||||
(self.uploader, 'docker.io/t/baz', 'rdo_release')
|
||||
])
|
||||
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
|
|
@ -388,7 +388,7 @@ class TestBaseImageUploader(base.TestCase):
|
|||
|
||||
def test_authenticate(self):
|
||||
req = self.requests
|
||||
auth = image_uploader.BaseImageUploader.authenticate
|
||||
auth = self.uploader.authenticate
|
||||
url1 = urlparse('docker://docker.io/t/nova-api:latest')
|
||||
|
||||
# no auth required
|
||||
|
|
@ -420,7 +420,7 @@ class TestBaseImageUploader(base.TestCase):
|
|||
|
||||
def test_authenticate_with_no_service(self):
|
||||
req = self.requests
|
||||
auth = image_uploader.BaseImageUploader.authenticate
|
||||
auth = self.uploader.authenticate
|
||||
url1 = urlparse('docker://docker.io/t/nova-api:latest')
|
||||
|
||||
headers = {
|
||||
|
|
@ -1003,6 +1003,68 @@ class TestPythonImageUploader(base.TestCase):
|
|||
target_session=target_session
|
||||
)
|
||||
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'PythonImageUploader.authenticate')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'PythonImageUploader._fetch_manifest')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'PythonImageUploader._cross_repo_mount')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'PythonImageUploader._copy_registry_to_registry')
|
||||
def test_insecure_registry(
|
||||
self, _copy_registry_to_registry, _cross_repo_mount,
|
||||
_fetch_manifest, authenticate):
|
||||
target_session = mock.Mock()
|
||||
source_session = mock.Mock()
|
||||
authenticate.side_effect = [
|
||||
target_session,
|
||||
source_session
|
||||
]
|
||||
manifest = json.dumps({
|
||||
'config': {
|
||||
'digest': 'sha256:1234',
|
||||
},
|
||||
'layers': [
|
||||
{'digest': 'sha256:aaa'},
|
||||
{'digest': 'sha256:bbb'},
|
||||
{'digest': 'sha256:ccc'}
|
||||
],
|
||||
})
|
||||
_fetch_manifest.return_value = manifest
|
||||
|
||||
image = '192.0.2.0:8787/tripleomaster/heat-docker-agents-centos'
|
||||
tag = 'latest'
|
||||
push_destination = 'localhost:8787'
|
||||
# push_image = 'localhost:8787/tripleomaster/heat-docker-agents-centos'
|
||||
task = image_uploader.UploadTask(
|
||||
image_name=image + ':' + tag,
|
||||
pull_source=None,
|
||||
push_destination=push_destination,
|
||||
append_tag=None,
|
||||
modify_role=None,
|
||||
modify_vars=None,
|
||||
dry_run=False,
|
||||
cleanup='full'
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
[],
|
||||
self.uploader.upload_image(task)
|
||||
)
|
||||
source_url = urlparse('docker://192.0.2.0:8787/tripleomaster/'
|
||||
'heat-docker-agents-centos:latest')
|
||||
target_url = urlparse('docker://localhost:8787/tripleomaster/'
|
||||
'heat-docker-agents-centos:latest')
|
||||
|
||||
authenticate.assert_has_calls([
|
||||
mock.call(
|
||||
target_url
|
||||
),
|
||||
mock.call(
|
||||
source_url
|
||||
),
|
||||
])
|
||||
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'PythonImageUploader.authenticate')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
|
|
|
|||
Loading…
Reference in New Issue