openstack
/
tripleo-common
Code Issues Proposed changes

Merge "Use config for layers"

This commit is contained in:
Zuul 2020-02-27 04:42:15 +00:00 committed by Gerrit Code Review
parent b1151922aa 2950738526
commit 62e28b2c09
2 changed files with 134 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
tripleo_common/image/image_uploader.py
View File

@ -2114,13 +2114,7 @@ class PythonImageUploader(BaseImageUploader):
name = '%s%s' % (source_url.netloc, source_url.path)
image, manifest, config_str = cls._image_manifest_config(name)
all_layers = cls._containers_json('overlay-layers', 'layers.json')
layers_by_digest = {}
for l in all_layers:
if 'diff-digest' in l:
layers_by_digest[l['diff-digest']] = l
if 'compressed-diff-digest' in l:
layers_by_digest[l['compressed-diff-digest']] = l
layers_by_digest = cls._get_all_local_layers_by_digest()
# Upload all layers
copy_jobs = []
@ -2180,6 +2174,50 @@ class PythonImageUploader(BaseImageUploader):
def _containers_json(cls, *path):
return json.loads(cls._containers_file(*path))
@classmethod
def _get_all_local_layers_by_digest(cls):
all_layers = cls._containers_json('overlay-layers', 'layers.json')
layers_by_digest = {}
for l in all_layers:
if 'diff-digest' in l:
layers_by_digest[l['diff-digest']] = l
if 'compressed-diff-digest' in l:
layers_by_digest[l['compressed-diff-digest']] = l
return layers_by_digest
@classmethod
def _get_local_layers_manifest(cls, manifest, config_str):
"""Return a valid local manifest
The manifest that is kept in the container storage is the
original manifest but the layers may be different once processed
by libpod & company. We want a valid manifest for the local
file system so we need to use the root fs layers from the container
config rather than just assuming the original manifest is still
valid.
"""
layers = cls._get_all_local_layers_by_digest()
config = json.loads(config_str)
rootfs = config.get('rootfs', {})
layer_ids = rootfs.get('diff_ids', None)
if not layer_ids:
# TODO(aschultz): add container name/path
LOG.warning('Container missing rootfs layers')
return manifest
# clear out the manifest layers
manifest['layers'] = []
for layer in layer_ids:
layer_digest = {'mediaType': MEDIA_BLOB}
if layer not in layers:
raise ImageNotFoundException('Unable to find layer %s in the '
'local layers' % layer)
layer_digest['digest'] = layer
# podman currently doesn't do compressed layers so just use
# the diff-size
layer_digest['size'] = layers[layer]['diff-size']
manifest['layers'].append(layer_digest)
return manifest
@classmethod
def _image_manifest_config(cls, name):
image = None
@ -2201,6 +2239,7 @@ class PythonImageUploader(BaseImageUploader):
six.b(config_digest)).decode("utf-8")
config_str = cls._containers_file('overlay-images', image_id,
config_id)
manifest = cls._get_local_layers_manifest(manifest, config_str)
manifest['config']['size'] = len(config_str)
manifest['config']['mediaType'] = MEDIA_CONFIG
return image, manifest, config_str

89
tripleo_common/tests/image/test_image_uploader.py
View File

@ -2469,11 +2469,14 @@ class TestPythonImageUploader(base.TestCase):
'overlay-layers', 'layers.json')
)
@mock.patch('tripleo_common.image.image_uploader.'
'PythonImageUploader._get_local_layers_manifest')
@mock.patch('tripleo_common.image.image_uploader.'
'PythonImageUploader._containers_json')
@mock.patch('tripleo_common.image.image_uploader.'
'PythonImageUploader._containers_file')
def test_image_manifest_config(self, _containers_file, _containers_json):
def test_image_manifest_config(self, _containers_file, _containers_json,
_get_local_layers_manifest):
_containers_file.return_value = '{"config": {}}'
images_not_found = [{
'id': 'aaaa',
@ -2495,6 +2498,7 @@ class TestPythonImageUploader(base.TestCase):
'layers': [],
}
_containers_json.side_effect = [images_not_found, images, man]
_get_local_layers_manifest.return_value = man
self.assertRaises(
ImageNotFoundException,
@ -2516,6 +2520,89 @@ class TestPythonImageUploader(base.TestCase):
_containers_file.assert_called_once_with(
'overlay-images', 'cccc', '=c2hhMjU2OjEyMzQ='
)
_get_local_layers_manifest.assert_called_once_with(
man, config_str)
@mock.patch('tripleo_common.image.image_uploader.'
'PythonImageUploader._get_all_local_layers_by_digest')
def test_get_local_layers_manifest(self, mock_layers_by_digest):
mock_layers_by_digest.return_value = {
'sha256:1': {'diff-size': 8},
'sha256:2': {'diff-size': 9}
}
man = {
'config': {
'digest': 'sha256:1234',
'size': 2,
'mediaType': 'application/vnd.docker.container.image.v1+json'
},
'layers': [
{"digest": "sha256:12345"}
]
}
config_str = json.dumps({'rootfs':
{'diff_ids': ['sha256:1', 'sha256:2']}})
manifest = self.uploader._get_local_layers_manifest(man, config_str)
manifest_expected = {
'config': man['config'],
'layers': [
{'digest': 'sha256:1',
'mediaType': 'application/vnd.docker.image.rootfs.diff.tar',
'size': 8},
{'digest': 'sha256:2',
'mediaType': 'application/vnd.docker.image.rootfs.diff.tar',
'size': 9}
]
}
self.assertEqual(manifest_expected, manifest)
@mock.patch('tripleo_common.image.image_uploader.'
'PythonImageUploader._get_all_local_layers_by_digest')
def test_get_local_layers_manifest_missing_rootfs(self,
mock_layers_by_digest):
mock_layers_by_digest.return_value = {
'sha256:1': {'diff-size': 8}
}
man = {
'config': {
'digest': 'sha256:1234',
'size': 2,
'mediaType': 'application/vnd.docker.container.image.v1+json'
},
'layers': [
{"digest": "sha256:12345"}
]
}
manifest = self.uploader._get_local_layers_manifest(man, '{}')
self.assertEqual(man, manifest)
@mock.patch('tripleo_common.image.image_uploader.'
'PythonImageUploader._get_all_local_layers_by_digest')
def test_get_local_layers_manifest_missing_layer(self,
mock_layers_by_digest):
mock_layers_by_digest.return_value = {
'sha256:1': {'diff-size': 8}
}
man = {
'config': {
'digest': 'sha256:1234',
'size': 2,
'mediaType': 'application/vnd.docker.container.image.v1+json'
},
'layers': [
{"digest": "sha256:12345"}
]
}
config_str = json.dumps({'rootfs':
{'diff_ids': ['sha256:3']}})
self.assertRaises(ImageNotFoundException,
self.uploader._get_local_layers_manifest,
man,
config_str)
@mock.patch('tripleo_common.image.image_uploader.'
'PythonImageUploader._image_manifest_config')