From 5304c3432498ada824dc5f564f527aa360f21df8 Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Wed, 13 Nov 2019 14:53:58 +1300 Subject: [PATCH] Local file based image uploader This change modifies the "openstack overcloud image upload" to add a --local argument which will copy the files to /var/lib/ironic/images instead of uploading them to glance. /var/lib/ironic is chosen as it is bind-mounted into the ironic containers on the undercloud. The on-disk layout is prefixed by the architecture (and optionally the platform) for example: /var/lib/ironic/images/x86_64/overcloud-full.qcow2 /var/lib/ironic/images/x86_64/overcloud-full.initrd /var/lib/ironic/images/x86_64/overcloud-full.vmlinuz /var/lib/ironic/images/power9-ppc64le/overcloud-full.qcow2 When glance is switched off this will become the default way of preparing image files for deployment. Blueprint: nova-less-deploy Change-Id: I4e1b8a7effca21b5aa054a78ddeaae7338ca9d2c --- tripleoclient/constants.py | 1 + .../overcloud_image/test_overcloud_image.py | 240 ++++++++++++++++++ tripleoclient/v1/overcloud_image.py | 157 +++++++++++- 3 files changed, 389 insertions(+), 9 deletions(-) diff --git a/tripleoclient/constants.py b/tripleoclient/constants.py index 3018bac00..bd8e54e26 100644 --- a/tripleoclient/constants.py +++ b/tripleoclient/constants.py @@ -47,6 +47,7 @@ PUPPET_BASE = "/etc/puppet/" STACK_TIMEOUT = 240 IRONIC_HTTP_BOOT_BIND_MOUNT = '/var/lib/ironic/httpboot' +IRONIC_LOCAL_IMAGE_PATH = '/var/lib/ironic/images' # The default ffwd upgrade ansible playbooks generated from heat stack output FFWD_UPGRADE_PLAYBOOK = "fast_forward_upgrade_playbook.yaml" diff --git a/tripleoclient/tests/v1/overcloud_image/test_overcloud_image.py b/tripleoclient/tests/v1/overcloud_image/test_overcloud_image.py index 1ee8d118d..5909f7608 100644 --- a/tripleoclient/tests/v1/overcloud_image/test_overcloud_image.py +++ b/tripleoclient/tests/v1/overcloud_image/test_overcloud_image.py @@ -21,6 +21,7 @@ import tripleo_common.arch from tripleoclient.tests import base from tripleoclient.tests.fakes import FakeHandle from tripleoclient.tests.v1.test_plugin import TestPluginV1 +from tripleoclient import utils as plugin_utils from tripleoclient.v1 import overcloud_image @@ -150,6 +151,245 @@ class TestBaseClientAdapter(base.TestCommand): mock_copy_file.assert_called_once_with( self.adapter, 'discimg', 'discimgprod') + @mock.patch('subprocess.check_call', autospec=True) + def test_copy_file(self, mock_subprocess_call): + self.adapter._copy_file('/foo.qcow2', 'bar.qcow2') + mock_subprocess_call.assert_called_once_with( + 'sudo cp -f "/foo.qcow2" "bar.qcow2"', shell=True) + + @mock.patch('subprocess.check_call', autospec=True) + def test_move_file(self, mock_subprocess_call): + self.adapter._move_file('/foo.qcow2', 'bar.qcow2') + mock_subprocess_call.assert_called_once_with( + 'sudo mv "/foo.qcow2" "bar.qcow2"', shell=True) + + @mock.patch('subprocess.check_call', autospec=True) + def test_make_dirs(self, mock_subprocess_call): + self.adapter._make_dirs('/foo/bar/baz') + mock_subprocess_call.assert_called_once_with( + 'sudo mkdir -m 0775 -p "/foo/bar/baz"', shell=True) + + +class TestFileImageClientAdapter(TestPluginV1): + + def setUp(self): + super(TestFileImageClientAdapter, self).setUp() + self.updated = [] + self.adapter = overcloud_image.FileImageClientAdapter( + image_path='/home/foo', + local_path='/my/images', + updated=self.updated + ) + self.image = mock.Mock() + self.image.id = 'file:///my/images/x86_64/overcloud-full.qcow2' + self.image.name = 'overcloud-full' + self.image.checksum = 'asdf' + self.image.created_at = '2019-11-14T01:33:39' + self.image.size = 982802432 + + @mock.patch('os.path.exists') + def test_get_image_property(self, mock_exists): + mock_exists.side_effect = [ + True, True, False, False + ] + image = mock.Mock() + image.id = 'file:///my/images/x86_64/overcloud-full.qcow2' + # file exists + self.assertEqual( + 'file:///my/images/x86_64/overcloud-full.vmlinuz', + self.adapter.get_image_property(image, 'kernel_id') + ) + self.assertEqual( + 'file:///my/images/x86_64/overcloud-full.initrd', + self.adapter.get_image_property(image, 'ramdisk_id') + ) + # file doesn't exist + self.assertIsNone( + self.adapter.get_image_property(image, 'kernel_id') + ) + self.assertIsNone( + self.adapter.get_image_property(image, 'ramdisk_id') + ) + self.assertRaises(ValueError, self.adapter.get_image_property, + image, 'foo') + + def test_paths(self): + self.assertEqual( + ('/my/images/x86_64', + 'overcloud-full.vmlinuz'), + self.adapter._paths( + 'overcloud-full', + plugin_utils.overcloud_kernel, + 'x86_64', + None + ) + ) + self.assertEqual( + ('/my/images', + 'overcloud-full.qcow2'), + self.adapter._paths( + 'overcloud-full', + plugin_utils.overcloud_image, + None, + None + ) + ) + self.assertEqual( + ('/my/images/power9-ppc64le', + 'overcloud-full.initrd'), + self.adapter._paths( + 'overcloud-full', + plugin_utils.overcloud_ramdisk, + 'ppc64le', + 'power9' + ) + ) + + @mock.patch('os.path.exists') + @mock.patch('os.stat') + @mock.patch('tripleoclient.utils.file_checksum') + def test_get_image(self, mock_checksum, mock_stat, mock_exists): + mock_exists.return_value = True + mock_stat.return_value.st_size = 982802432 + mock_stat.return_value.st_mtime = 1573695219 + mock_checksum.return_value = 'asdf' + + image = self.adapter._get_image( + '/my/images/x86_64/overcloud-full.qcow2') + self.assertEqual( + 'file:///my/images/x86_64/overcloud-full.qcow2', + image.id + ) + self.assertEqual('overcloud-full', image.name) + self.assertEqual('asdf', image.checksum) + self.assertEqual('2019-11-14T01:33:39', image.created_at) + self.assertEqual(982802432, image.size) + + @mock.patch('tripleoclient.utils.file_checksum') + @mock.patch('tripleoclient.v1.overcloud_image.' + 'FileImageClientAdapter._get_image', autospec=True) + @mock.patch('tripleoclient.v1.overcloud_image.' + 'BaseClientAdapter._move_file', autospec=True) + def test_image_try_update(self, mock_move, mock_get_image, mock_checksum): + + # existing image with identical checksum + mock_checksum.return_value = 'asdf' + mock_get_image.return_value = self.image + self.assertEqual( + self.image, + self.adapter._image_try_update( + '/home/foo/overcloud-full.qcow2', + '/my/images/x86_64/overcloud-full.qcow2' + ) + ) + self.assertEqual([], self.updated) + + # no image to update + mock_get_image.return_value = None + self.assertIsNone( + self.adapter._image_try_update( + '/home/foo/overcloud-full.qcow2', + '/my/images/x86_64/overcloud-full.qcow2' + ) + ) + self.assertEqual([], self.updated) + + # existing image with different checksum, but update_existing=False + mock_checksum.return_value = 'fdsa' + mock_get_image.return_value = self.image + self.assertEqual( + self.image, + self.adapter._image_try_update( + '/home/foo/overcloud-full.qcow2', + '/my/images/x86_64/overcloud-full.qcow2' + ) + ) + self.assertEqual([], self.updated) + + # existing image with different checksum, update_existing=True + self.adapter.update_existing = True + self.assertIsNone( + self.adapter._image_try_update( + '/home/foo/overcloud-full.qcow2', + '/my/images/x86_64/overcloud-full.qcow2' + ) + ) + self.assertEqual( + ['/my/images/x86_64/overcloud-full.qcow2'], + self.updated + ) + mock_move.assert_called_once_with( + self.adapter, + '/my/images/x86_64/overcloud-full.qcow2', + '/my/images/x86_64/overcloud-full_20191114T013339.qcow2' + ) + + @mock.patch('subprocess.check_call', autospec=True) + @mock.patch('tripleoclient.v1.overcloud_image.' + 'FileImageClientAdapter._get_image', autospec=True) + @mock.patch('os.path.isdir') + def test_upload_image(self, mock_isdir, mock_get_image, + mock_subprocess_call): + mock_isdir.return_value = False + + mock_get_image.return_value = self.image + + result = self.adapter._upload_image( + '/home/foo/overcloud-full.qcow2', + '/my/images/x86_64/overcloud-full.qcow2' + ) + self.assertEqual(self.image, result) + mock_subprocess_call.assert_has_calls([ + mock.call('sudo mkdir -m 0775 -p "/my/images/x86_64"', shell=True), + mock.call('sudo cp -f "/home/foo/overcloud-full.qcow2" ' + '"/my/images/x86_64/overcloud-full.qcow2"', shell=True) + ]) + + @mock.patch('tripleoclient.v1.overcloud_image.' + 'FileImageClientAdapter._upload_image', autospec=True) + @mock.patch('tripleoclient.v1.overcloud_image.' + 'FileImageClientAdapter._image_try_update', autospec=True) + def test_update_or_upload(self, mock_image_try_update, mock_upload_image): + + # image exists + mock_image_try_update.return_value = self.image + self.assertEqual( + self.image, + self.adapter.update_or_upload( + image_name='overcloud-full', + properties={}, + names_func=plugin_utils.overcloud_image, + arch='x86_64' + ) + ) + mock_upload_image.assert_not_called() + + # image needs uploading + mock_image_try_update.return_value = None + mock_upload_image.return_value = self.image + self.assertEqual( + self.image, + self.adapter.update_or_upload( + image_name='overcloud-full', + properties={}, + names_func=plugin_utils.overcloud_image, + arch='x86_64' + ) + ) + mock_upload_image.assert_called_once_with( + self.adapter, + '/home/foo/overcloud-full.qcow2', + '/my/images/x86_64/overcloud-full.qcow2' + ) + mock_image_try_update.assert_has_calls([ + mock.call(self.adapter, + '/home/foo/overcloud-full.qcow2', + '/my/images/x86_64/overcloud-full.qcow2'), + mock.call(self.adapter, + '/home/foo/overcloud-full.qcow2', + '/my/images/x86_64/overcloud-full.qcow2') + ]) + class TestGlanceClientAdapter(TestPluginV1): diff --git a/tripleoclient/v1/overcloud_image.py b/tripleoclient/v1/overcloud_image.py index 0e47fe1eb..3a0b92e7e 100644 --- a/tripleoclient/v1/overcloud_image.py +++ b/tripleoclient/v1/overcloud_image.py @@ -16,6 +16,8 @@ from __future__ import print_function import abc +import collections +from datetime import datetime import logging import os import re @@ -118,8 +120,19 @@ class BaseClientAdapter(object): pass def _copy_file(self, src, dest): - subprocess.check_call('sudo cp -f "{0}" "{1}"'.format(src, dest), - shell=True) + cmd = 'sudo cp -f "{0}" "{1}"'.format(src, dest) + self.log.debug(cmd) + subprocess.check_call(cmd, shell=True) + + def _move_file(self, src, dest): + cmd = 'sudo mv "{0}" "{1}"'.format(src, dest) + self.log.debug(cmd) + subprocess.check_call(cmd, shell=True) + + def _make_dirs(self, path): + cmd = 'sudo mkdir -m 0775 -p "{0}"'.format(path) + self.log.debug(cmd) + subprocess.check_call(cmd, shell=True) def _files_changed(self, filepath1, filepath2): return (plugin_utils.file_checksum(filepath1) != @@ -154,6 +167,115 @@ class BaseClientAdapter(object): return file_descriptor +class FileImageClientAdapter(BaseClientAdapter): + + def __init__(self, local_path, **kwargs): + super(FileImageClientAdapter, self).__init__(**kwargs) + self.local_path = local_path + + def get_image_property(self, image, prop): + if prop == 'kernel_id': + path = os.path.splitext(image.id)[0] + '.vmlinuz' + if os.path.exists(path): + return path + return None + elif prop == 'ramdisk_id': + path = os.path.splitext(image.id)[0] + '.initrd' + if os.path.exists(path): + return path + return None + raise ValueError('Unsupported property %s' % prop) + + def _print_image_info(self, image): + table = PrettyTable(['Path', 'Name', 'Size']) + table.add_row([image.id, image.name, image.size]) + print(table, file=sys.stdout) + + def _paths(self, image_name, names_func, arch, platform): + (arch_path, extension) = names_func( + image_name, arch=arch, platform=platform, use_subdir=True) + image_file = image_name + extension + + dest_dir = os.path.split( + os.path.join(self.local_path, arch_path))[0] + return (dest_dir, image_file) + + def _get_image(self, path): + if not os.path.exists(path): + return + stat = os.stat(path) + created_at = datetime.fromtimestamp( + stat.st_mtime).isoformat() + + Image = collections.namedtuple( + 'Image', + 'id, name, checksum, created_at, size' + ) + (dir_path, filename) = os.path.split(path) + (name, extension) = os.path.splitext(filename) + checksum = plugin_utils.file_checksum(path) + + return Image( + id='file://%s' % path, + name=name, + checksum=checksum, + created_at=created_at, + size=stat.st_size + ) + + def _image_changed(self, image, filename): + return image.checksum != plugin_utils.file_checksum(filename) + + def _image_try_update(self, src_path, dest_path): + image = self._get_image(dest_path) + if image: + if self._image_changed(image, src_path): + if self.update_existing: + dest_base, dest_ext = os.path.splitext(dest_path) + dest_datestamp = re.sub( + r'[\-:\.]|(0+$)', '', image.created_at) + dest_mv = dest_base + '_' + dest_datestamp + dest_ext + self._move_file(dest_path, dest_mv) + + if self.updated is not None: + self.updated.append(dest_path) + return None + else: + print('Image "%s" already exists and can be updated' + ' with --update-existing.' % dest_path) + return image + else: + print('Image "%s" is up-to-date, skipping.' % dest_path) + return image + else: + return None + + def _upload_image(self, src_path, dest_path): + dest_dir = os.path.split(dest_path)[0] + if not os.path.isdir(dest_dir): + self._make_dirs(dest_dir) + + self._copy_file(src_path, dest_path) + image = self._get_image(dest_path) + print('Image "%s" was copied.' % image.id, file=sys.stdout) + self._print_image_info(image) + return image + + def update_or_upload(self, image_name, properties, names_func, + arch, platform=None, + disk_format='qcow2', container_format='bare'): + (dest_dir, image_file) = self._paths( + image_name, names_func, arch, platform) + + src_path = os.path.join(self.image_path, image_file) + dest_path = os.path.join(dest_dir, image_file) + existing_image = self._image_try_update(src_path, dest_path) + if existing_image: + return existing_image + + return self._upload_image(src_path, dest_path) + + class GlanceClientAdapter(BaseClientAdapter): def __init__(self, client, **kwargs): @@ -263,13 +385,15 @@ class UploadOvercloudImage(command.Command): log = logging.getLogger(__name__ + ".UploadOvercloudImage") def _get_client_adapter(self, parsed_args): - return GlanceClientAdapter( - self.app.client_manager.image, - progress=parsed_args.progress, - image_path=parsed_args.image_path, - update_existing=parsed_args.update_existing, - updated=self.updated - ) + kwargs = { + 'progress': parsed_args.progress, + 'image_path': parsed_args.image_path, + 'update_existing': parsed_args.update_existing, + 'updated': self.updated + } + if parsed_args.local: + return FileImageClientAdapter(parsed_args.local_path, **kwargs) + return GlanceClientAdapter(self.app.client_manager.image, **kwargs) def _get_environment_var(self, envvar, default, deprecated=[]): for env_key in deprecated: @@ -353,6 +477,21 @@ class UploadOvercloudImage(command.Command): action="store_true", default=False, help=_('Show progress bar for upload files action')) + parser.add_argument( + "--local", + dest="local", + action="store_true", + default=False, + help=_('Copy files locally, even if there is an image service ' + 'endpoint')) + parser.add_argument( + "--local-path", + default=self._get_environment_var( + 'LOCAL_IMAGE_PATH', + constants.IRONIC_LOCAL_IMAGE_PATH), + help=_("Root directory for image file copy destination when there " + "is no image endpoint, or when --local is specified") + ) return parser