Added image utils
Added set of image utilities so as to make it possible to deal with images when they are available via http or when they are local files, when they are packed into containers or not, etc. Change-Id: Id03f6bcaed2a61b4c6d644675ef4cfeb353258a9 Implements: blueprint image-based-provisioning
This commit is contained in:
Vladimir Kozhukalov
Alexander Gordeev
parent
291ef045f5
commit
a6d5da023d
|
|
@ -209,11 +209,14 @@ class Nailgun(object):
|
|||
def image_scheme(self, partition_scheme):
|
||||
data = self.data
|
||||
image_scheme = objects.ImageScheme()
|
||||
root_image_uri = 'http://%s/targetimages/%s.img.gz' % (
|
||||
data['ks_meta']['master_ip'],
|
||||
data['profile'].split('_')[0]
|
||||
)
|
||||
image_scheme.add_image(
|
||||
uri=data['ks_meta']['image_uri'],
|
||||
uri=root_image_uri,
|
||||
target_device=partition_scheme.root_device(),
|
||||
image_format=data['ks_meta']['image_format'],
|
||||
container=data['ks_meta']['image_container'],
|
||||
size=data['ks_meta'].get('image_size'),
|
||||
image_format='ext4',
|
||||
container='gzip',
|
||||
)
|
||||
return image_scheme
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from oslo.config import cfg
|
|||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.utils import fs_utils as fu
|
||||
from fuel_agent.utils import img_utils as iu
|
||||
from fuel_agent.utils import lvm_utils as lu
|
||||
from fuel_agent.utils import md_utils as mu
|
||||
from fuel_agent.utils import partition_utils as pu
|
||||
|
|
@ -128,11 +129,24 @@ class Manager(object):
|
|||
uri='file://%s' % CONF.config_drive_path,
|
||||
target_device=configdrive_device,
|
||||
image_format='iso9660',
|
||||
container='raw',
|
||||
container='raw'
|
||||
)
|
||||
|
||||
def do_copyimage(self):
|
||||
pass
|
||||
for image in self.image_scheme.images:
|
||||
processing = iu.Chain()
|
||||
processing.append(image.uri)
|
||||
|
||||
if image.uri.startswith('http://'):
|
||||
processing.append(iu.HttpUrl)
|
||||
elif image.uri.startswith('file://'):
|
||||
processing.append(iu.LocalFile)
|
||||
|
||||
if image.container == 'gzip':
|
||||
processing.append(iu.GunzipStream)
|
||||
|
||||
processing.append(image.target_device)
|
||||
processing.process()
|
||||
|
||||
def do_bootloader(self):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from fuel_agent import errors
|
|||
|
||||
|
||||
class Image(object):
|
||||
SUPPORTED_CONTAINERS = ['raw']
|
||||
SUPPORTED_CONTAINERS = ['raw', 'gzip']
|
||||
|
||||
def __init__(self, uri, target_device,
|
||||
image_format, container, size=None):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,97 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
from oslotest import base as test_base
|
||||
import requests
|
||||
import zlib
|
||||
|
||||
from fuel_agent.utils import img_utils as iu
|
||||
|
||||
|
||||
class TestTarget(test_base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestTarget, self).setUp()
|
||||
self.tgt = iu.Target()
|
||||
|
||||
def test_target_next(self):
|
||||
self.assertRaises(StopIteration, self.tgt.next)
|
||||
|
||||
@mock.patch.object(iu.Target, '__iter__')
|
||||
def test_target_target(self, mock_iter):
|
||||
mock_iter.return_value = iter(['chunk1', 'chunk2', 'chunk3'])
|
||||
m = mock.mock_open()
|
||||
with mock.patch('six.moves.builtins.open', m):
|
||||
self.tgt.target()
|
||||
|
||||
mock_write_expected_calls = [mock.call('chunk1'), mock.call('chunk2'),
|
||||
mock.call('chunk3')]
|
||||
file_handle = m()
|
||||
self.assertEqual(mock_write_expected_calls,
|
||||
file_handle.write.call_args_list)
|
||||
file_handle.flush.assert_called_once_with()
|
||||
|
||||
|
||||
class TestLocalFile(test_base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestLocalFile, self).setUp()
|
||||
self.lf = iu.LocalFile('/dev/null')
|
||||
|
||||
def test_localfile_next(self):
|
||||
self.lf.fileobj = mock.Mock()
|
||||
self.lf.fileobj.read.side_effect = ['some_data', 'another_data']
|
||||
self.assertEqual('some_data', self.lf.next())
|
||||
self.assertEqual('another_data', self.lf.next())
|
||||
self.assertRaises(StopIteration, self.lf.next)
|
||||
|
||||
|
||||
class TestHttpUrl(test_base.BaseTestCase):
|
||||
@mock.patch.object(requests, 'get')
|
||||
def test_httpurl_iter(self, mock_r_get):
|
||||
content = ['fake content #1', 'fake content #2']
|
||||
mock_r_get.return_value.iter_content.return_value = content
|
||||
httpurl = iu.HttpUrl('fake_url')
|
||||
for data in enumerate(httpurl):
|
||||
self.assertEqual(content[data[0]], data[1])
|
||||
self.assertEqual('fake_url', httpurl.url)
|
||||
|
||||
|
||||
class TestGunzipStream(test_base.BaseTestCase):
|
||||
def test_gunzip_stream_next(self):
|
||||
content = ['fake content #1']
|
||||
compressed_stream = [zlib.compress(data) for data in content]
|
||||
gunzip_stream = iu.GunzipStream(compressed_stream)
|
||||
for data in enumerate(gunzip_stream):
|
||||
self.assertEqual(content[data[0]], data[1])
|
||||
|
||||
|
||||
class TestChain(test_base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestChain, self).setUp()
|
||||
self.chain = iu.Chain()
|
||||
|
||||
def test_append(self):
|
||||
self.assertEqual(0, len(self.chain.processors))
|
||||
self.chain.append('fake_processor')
|
||||
self.assertIn('fake_processor', self.chain.processors)
|
||||
self.assertEqual(1, len(self.chain.processors))
|
||||
|
||||
def test_process(self):
|
||||
self.chain.processors.append('fake_uri')
|
||||
fake_processor = mock.Mock(spec=iu.Target)
|
||||
self.chain.processors.append(fake_processor)
|
||||
self.chain.processors.append('fake_target')
|
||||
self.chain.process()
|
||||
expected_calls = [mock.call('fake_uri')]
|
||||
self.assertEqual(expected_calls, fake_processor.call_args_list)
|
||||
|
|
@ -23,6 +23,7 @@ from fuel_agent.objects import partition
|
|||
from fuel_agent.tests import test_nailgun
|
||||
from fuel_agent.utils import fs_utils as fu
|
||||
from fuel_agent.utils import hardware_utils as hu
|
||||
from fuel_agent.utils import img_utils as iu
|
||||
from fuel_agent.utils import lvm_utils as lu
|
||||
from fuel_agent.utils import md_utils as mu
|
||||
from fuel_agent.utils import partition_utils as pu
|
||||
|
|
@ -168,3 +169,35 @@ class TestManager(test_base.BaseTestCase):
|
|||
mock_p_ps_cd.return_value = None
|
||||
self.assertRaises(errors.WrongPartitionSchemeError,
|
||||
self.mgr.do_configdrive)
|
||||
|
||||
@mock.patch.object(iu, 'GunzipStream')
|
||||
@mock.patch.object(iu, 'LocalFile')
|
||||
@mock.patch.object(iu, 'HttpUrl')
|
||||
@mock.patch.object(iu, 'Chain')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
@mock.patch.object(utils, 'render_and_save')
|
||||
@mock.patch.object(hu, 'list_block_devices')
|
||||
def test_do_copyimage(self, mock_lbd, mock_u_ras, mock_u_e, mock_iu_c,
|
||||
mock_iu_h, mock_iu_l, mock_iu_g):
|
||||
|
||||
class FakeChain(object):
|
||||
processors = []
|
||||
|
||||
def append(self, thing):
|
||||
self.processors.append(thing)
|
||||
|
||||
def process(self):
|
||||
pass
|
||||
|
||||
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
|
||||
mock_iu_c.return_value = FakeChain()
|
||||
self.mgr.do_parsing()
|
||||
self.mgr.do_configdrive()
|
||||
self.mgr.do_copyimage()
|
||||
imgs = self.mgr.image_scheme.images
|
||||
self.assertEqual(2, len(imgs))
|
||||
expected_processors_list = [imgs[0].uri, iu.HttpUrl, iu.GunzipStream,
|
||||
imgs[0].target_device, imgs[1].uri,
|
||||
iu.LocalFile, imgs[1].target_device]
|
||||
self.assertEqual(expected_processors_list,
|
||||
mock_iu_c.return_value.processors)
|
||||
|
|
|
|||
|
|
@ -65,9 +65,6 @@ PROVISION_SAMPLE_DATA = {
|
|||
"power_address": "10.20.0.253",
|
||||
"name_servers": "\"10.20.0.2\"",
|
||||
"ks_meta": {
|
||||
"image_uri": "proto://fake_image_uri",
|
||||
"image_format": "fake_image_format",
|
||||
"image_container": "raw",
|
||||
"timezone": "America/Los_Angeles",
|
||||
"master_ip": "10.20.0.2",
|
||||
"mco_enable": 1,
|
||||
|
|
@ -461,10 +458,14 @@ class TestNailgun(test_base.BaseTestCase):
|
|||
i_scheme = self.drv.image_scheme(p_scheme)
|
||||
self.assertEqual(1, len(i_scheme.images))
|
||||
img = i_scheme.images[0]
|
||||
self.assertEqual('raw', img.container)
|
||||
self.assertEqual('fake_image_format', img.image_format)
|
||||
self.assertEqual('gzip', img.container)
|
||||
self.assertEqual('ext4', img.image_format)
|
||||
self.assertEqual('/dev/mapper/os-root', img.target_device)
|
||||
self.assertEqual('proto://fake_image_uri', img.uri)
|
||||
self.assertEqual(
|
||||
'http://%s/targetimages/%s.img.gz' % (
|
||||
self.drv.data['ks_meta']['master_ip'],
|
||||
self.drv.data['profile'].split('_')[0]),
|
||||
img.uri)
|
||||
self.assertEqual(None, img.size)
|
||||
|
||||
def test_getlabel(self):
|
||||
|
|
|
|||
|
|
@ -11,3 +11,81 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import abc
|
||||
import requests
|
||||
import zlib
|
||||
|
||||
|
||||
class Target(object):
|
||||
__metaclass__ = abc.ABCMeta
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def next(self):
|
||||
raise StopIteration()
|
||||
|
||||
def target(self, filename='/dev/null'):
|
||||
with open(filename, 'wb') as f:
|
||||
for chunk in self:
|
||||
f.write(chunk)
|
||||
f.flush()
|
||||
|
||||
|
||||
class LocalFile(Target):
|
||||
def __init__(self, filename):
|
||||
self.filename = str(filename)
|
||||
self.fileobj = None
|
||||
|
||||
def next(self):
|
||||
if not self.fileobj:
|
||||
self.fileobj = open(self.filename, 'rb')
|
||||
buffer = self.fileobj.read(1048576)
|
||||
if buffer:
|
||||
return buffer
|
||||
else:
|
||||
self.fileobj.close()
|
||||
raise StopIteration()
|
||||
|
||||
|
||||
class HttpUrl(Target):
|
||||
def __init__(self, url):
|
||||
self.url = str(url)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(requests.get(self.url, stream=True).iter_content(1048576))
|
||||
|
||||
|
||||
class GunzipStream(Target):
|
||||
def __init__(self, stream):
|
||||
self.stream = iter(stream)
|
||||
#NOTE(agordeev): toggle automatic header detection on
|
||||
self.decompressor = zlib.decompressobj(zlib.MAX_WBITS | 32)
|
||||
|
||||
def next(self):
|
||||
try:
|
||||
return self.decompressor.decompress(self.stream.next())
|
||||
except StopIteration:
|
||||
raise
|
||||
|
||||
|
||||
class Chain(object):
|
||||
def __init__(self):
|
||||
self.processors = []
|
||||
|
||||
def append(self, processor):
|
||||
self.processors.append(processor)
|
||||
|
||||
def process(self):
|
||||
def jump(proc, next_proc):
|
||||
# if next_proc is just a string we assume it is a filename
|
||||
# and we save stream into a file
|
||||
if isinstance(next_proc, (str, unicode)):
|
||||
proc.target(next_proc)
|
||||
return LocalFile(next_proc)
|
||||
# if next_proc is not a string we return new instance
|
||||
# initialized with the previous one
|
||||
else:
|
||||
return next_proc(proc)
|
||||
return reduce(jump, self.processors)
|
||||
|
|
|
|||
|
|
@ -7,3 +7,4 @@ six>=1.5.2
|
|||
pbr>=0.7.0
|
||||
Jinja2
|
||||
stevedore>=0.15
|
||||
requests>=1.2.3
|
||||
|
|
|
|||
Loading…
Reference in New Issue