openstack
/
fuel-agent
Code Issues Proposed changes

Add processing of keep_data flag 

In case when some partition has keep_data = True
need to elevate flag to file system level and
do not to create new partition table. Do not clean
file system when it has keep_data = True.
There should not be a problem if some raw data
will not be cleaned.

Change-Id: Ifccc9256b12944283219cffeedb8c8da8ab420f8
Implements: blueprint partition-preservation
This commit is contained in:
Artur Svechnikov 2015-07-14 16:08:22 +03:00
parent 9e22866f3e
commit 68494fb0dd
7 changed files with 217 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
fuel_agent/drivers/nailgun.py
View File

@ -262,7 +262,9 @@ class Nailgun(BaseDataDriver):
if volume.get('mount') != '/boot':
LOG.debug('Adding partition on disk %s: size=%s' %
(disk['name'], volume['size']))
prt = parted.add_partition(size=volume['size'])
prt = parted.add_partition(
size=volume['size'],
keep_data=volume.get('keep_data', False))
LOG.debug('Partition name: %s' % prt.name)
elif volume.get('mount') == '/boot' \
@ -275,7 +277,9 @@ class Nailgun(BaseDataDriver):
# huge disks if it is possible.
LOG.debug('Adding /boot partition on disk %s: '
'size=%s', disk['name'], volume['size'])
prt = parted.add_partition(size=volume['size'])
prt = parted.add_partition(
size=volume['size'],
keep_data=volume.get('keep_data', False))
LOG.debug('Partition name: %s', prt.name)
self._boot_partition_done = True
else:
@ -388,6 +392,7 @@ class Nailgun(BaseDataDriver):
fs_type=volume.get('file_system', 'xfs'),
fs_label=self._getlabel(volume.get('disk_label')))
partition_scheme.elevate_keep_data()
return partition_scheme
def parse_configdrive_scheme(self):

19
fuel_agent/manager.py
View File

@ -98,8 +98,27 @@ class Manager(object):
def __init__(self, data):
self.driver = utils.get_driver(CONF.data_driver)(data)
def do_clean_filesystems(self):
# All fs without keep_data flag and without image should be cleaned,
# mkfs will clean file table and such flags as --force are not needed
# TODO(asvechnikov): need to refactor processing keep_flag logic when
# data model will become flat
for fs in self.driver.partition_scheme.fss:
found_images = [img for img in self.driver.image_scheme.images
if img.target_device == fs.device]
if not fs.keep_data and not found_images:
fu.make_fs(fs.type, fs.options, fs.label, fs.device)
def do_partitioning(self):
LOG.debug('--- Partitioning disks (do_partitioning) ---')
if self.driver.partition_scheme.skip_partitioning:
LOG.debug('Some of fs has keep_data flag, '
'partitioning is skiping')
self.do_clean_filesystems()
return
# If disks are not wiped out at all, it is likely they contain lvm
# and md metadata which will prevent re-creating a partition table
# with 'device is busy' error.

6
fuel_agent/objects/base.py
View File

@ -17,8 +17,6 @@
import abc
import six
from fuel_agent.utils.decorators import abstractclassmethod
@six.add_metaclass(abc.ABCMeta)
class Serializable(object):
@ -27,6 +25,6 @@ class Serializable(object):
def to_dict(self):
pass
@abstractclassmethod
@classmethod
def from_dict(cls, data):
pass
return cls(**data)

113
fuel_agent/objects/partition.py
View File

@ -102,6 +102,11 @@ class Parted(base.Serializable):
separator = 'p'
return '%s%s%s' % (self.name, separator, self.next_count())
def partition_by_name(self, name):
found = filter(lambda x: (x.name == name), self.partitions)
if found:
return found[0]
def to_dict(self):
partitions = [partition.to_dict() for partition in self.partitions]
return {
@ -123,7 +128,8 @@ class Parted(base.Serializable):
class Partition(base.Serializable):
def __init__(self, name, count, device, begin, end, partition_type,
flags=None, guid=None, configdrive=False):
flags=None, guid=None, configdrive=False, keep_data=False):
self.keep_data = keep_data
self.name = name
self.count = count
self.device = device
@ -152,16 +158,15 @@ class Partition(base.Serializable):
'flags': self.flags,
'guid': self.guid,
'configdrive': self.configdrive,
'keep_data': self.keep_data,
}
@classmethod
def from_dict(cls, data):
return cls(**data)
class PhysicalVolume(base.Serializable):
def __init__(self, name, metadatasize=16, metadatacopies=2):
def __init__(self, name, metadatasize=16,
metadatacopies=2, keep_data=False):
self.keep_data = keep_data
self.name = name
self.metadatasize = metadatasize
self.metadatacopies = metadatacopies
@ -171,19 +176,17 @@ class PhysicalVolume(base.Serializable):
'name': self.name,
'metadatasize': self.metadatasize,
'metadatacopies': self.metadatacopies,
'keep_data': self.keep_data,
}
@classmethod
def from_dict(cls, data):
return cls(**data)
PV = PhysicalVolume
class VolumeGroup(base.Serializable):
def __init__(self, name, pvnames=None):
def __init__(self, name, pvnames=None, keep_data=False):
self.keep_data = keep_data
self.name = name
self.pvnames = pvnames or []
@ -194,20 +197,18 @@ class VolumeGroup(base.Serializable):
def to_dict(self):
return {
'name': self.name,
'pvnames': self.pvnames
'pvnames': self.pvnames,
'keep_data': self.keep_data,
}
@classmethod
def from_dict(cls, data):
return cls(**data)
VG = VolumeGroup
class LogicalVolume(base.Serializable):
def __init__(self, name, vgname, size):
def __init__(self, name, vgname, size, keep_data=False):
self.keep_data = keep_data
self.name = name
self.vgname = vgname
self.size = size
@ -222,12 +223,9 @@ class LogicalVolume(base.Serializable):
'name': self.name,
'vgname': self.vgname,
'size': self.size,
'keep_data': self.keep_data,
}
@classmethod
def from_dict(cls, data):
return cls(**data)
LV = LogicalVolume
@ -235,7 +233,8 @@ LV = LogicalVolume
class MultipleDevice(base.Serializable):
def __init__(self, name, level,
devices=None, spares=None):
devices=None, spares=None, keep_data=False):
self.keep_data = keep_data
self.name = name
self.level = level
self.devices = devices or []
@ -261,20 +260,18 @@ class MultipleDevice(base.Serializable):
'level': self.level,
'devices': self.devices,
'spares': self.spares,
'keep_data': self.keep_data,
}
@classmethod
def from_dict(cls, data):
return cls(**data)
MD = MultipleDevice
class FileSystem(base.Serializable):
def __init__(self, device, mount=None,
fs_type=None, fs_options=None, fs_label=None):
def __init__(self, device, mount=None, fs_type=None,
fs_options=None, fs_label=None, keep_data=False):
self.keep_data = keep_data
self.device = device
self.mount = mount
self.type = fs_type or 'xfs'
@ -288,12 +285,9 @@ class FileSystem(base.Serializable):
'fs_type': self.type,
'fs_options': self.options,
'fs_label': self.label,
'keep_data': self.keep_data,
}
@classmethod
def from_dict(cls, data):
return cls(**data)
FS = FileSystem
@ -377,6 +371,11 @@ class PartitionScheme(object):
'try to generate md name manually')
count += 1
def partition_by_name(self, name):
return next((parted.partition_by_name(name)
for parted in self.parteds
if parted.partition_by_name(name)), None)
def vg_by_name(self, vgname):
found = filter(lambda x: (x.name == vgname), self.vgs)
if found:
@ -472,6 +471,56 @@ class PartitionScheme(object):
if prt.configdrive:
return prt.name
def elevate_keep_data(self):
LOG.debug('Elevate keep_data flag from partitions')
for vg in self.vgs:
for pvname in vg.pvnames:
partition = self.partition_by_name(pvname)
if partition and partition.keep_data:
partition.keep_data = False
vg.keep_data = True
LOG.debug('Set keep_data to vg=%s' % vg.name)
for lv in self.lvs:
vg = self.vg_by_name(lv.vgname)
if vg.keep_data:
lv.keep_data = True
# Need to loop over lv again to remove keep flag from vg
for lv in self.lvs:
vg = self.vg_by_name(lv.vgname)
if vg.keep_data and lv.keep_data:
vg.keep_data = False
for fs in self.fss:
lv = self.lv_by_device_name(fs.device)
if lv:
if lv.keep_data:
lv.keep_data = False
fs.keep_data = True
LOG.debug('Set keep_data to fs=%s from lv=%s' %
(fs.mount, lv.name))
continue
partition = self.partition_by_name(fs.device)
if partition and partition.keep_data:
partition.keep_data = False
fs.keep_data = True
LOG.debug('Set keep flag to fs=%s from partition=%s' %
(fs.mount, partition.name))
@property
def skip_partitioning(self):
if any(fs.keep_data for fs in self.fss):
return True
if any(lv.keep_data for lv in self.lvs):
return True
if any(vg.keep_data for vg in self.vgs):
return True
for parted in self.parteds:
if any(prt.keep_data for prt in parted.partitions):
return True
def to_dict(self):
return {
'parteds': [parted.to_dict() for parted in self.parteds],

24
fuel_agent/tests/test_manager.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import mock
import os
import signal
@ -146,6 +147,29 @@ class TestManager(test_base.BaseTestCase):
mock_utils.makedirs_if_not_exists.assert_called_once_with(
'/tmp/target/etc/nailgun-agent')
@mock.patch('yaml.load')
@mock.patch.object(utils, 'init_http_request')
@mock.patch.object(hu, 'list_block_devices')
@mock.patch.object(fu, 'make_fs')
def test_do_partitioning_with_keep_data_flag(self, mock_fu_mf, mock_lbd,
mock_http, mock_yaml):
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
data = copy.deepcopy(test_nailgun.PROVISION_SAMPLE_DATA)
for disk in data['ks_meta']['pm_data']['ks_spaces']:
for volume in disk['volumes']:
if volume['type'] == 'pv' and volume['vg'] == 'image':
volume['keep_data'] = True
self.mgr = manager.Manager(data)
self.mgr.do_partitioning()
mock_fu_mf_expected_calls = [
mock.call('ext2', '', '', '/dev/sda3'),
mock.call('ext2', '', '', '/dev/sda4'),
mock.call('swap', '', '', '/dev/mapper/os-swap')]
self.assertEqual(mock_fu_mf_expected_calls, mock_fu_mf.call_args_list)
@mock.patch('six.moves.builtins.open')
@mock.patch.object(os, 'symlink')
@mock.patch.object(os, 'remove')

34
fuel_agent/tests/test_nailgun.py
View File

@ -426,7 +426,8 @@ SINGLE_DISK_KS_SPACES = [
"size": 200,
"type": "partition",
"file_system": "ext4",
"name": "Root"
"name": "Root",
"keep_data": True
},
],
"type": "disk",
@ -951,6 +952,37 @@ class TestNailgun(test_base.BaseTestCase):
drv.partition_scheme.fs_by_mount('/boot').device,
'/dev/sda3')
@mock.patch('fuel_agent.drivers.nailgun.yaml.load')
@mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
@mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')
def test_elevate_keep_data_single_disk(self, mock_lbd,
mock_http_req, mock_yaml):
data = copy.deepcopy(PROVISION_SAMPLE_DATA)
data['ks_meta']['pm_data']['ks_spaces'] = SINGLE_DISK_KS_SPACES
mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
drv = nailgun.Nailgun(data)
self.assertTrue(drv.partition_scheme.fs_by_mount('/').keep_data)
for parted in drv.partition_scheme.parteds:
for partition in parted.partitions:
self.assertFalse(partition.keep_data)
for md in drv.partition_scheme.mds:
self.assertFalse(md.keep_data)
for pv in drv.partition_scheme.pvs:
self.assertFalse(pv.keep_data)
for vg in drv.partition_scheme.vgs:
self.assertFalse(vg.keep_data)
for lv in drv.partition_scheme.lvs:
self.assertFalse(lv.keep_data)
for fs in drv.partition_scheme.fss:
if fs.mount != '/':
self.assertFalse(fs.keep_data)
@mock.patch('fuel_agent.drivers.nailgun.yaml.load')
@mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request')
@mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices')

53
fuel_agent/tests/test_partition.py
View File

@ -69,6 +69,7 @@ class TestMultipleDevice(unittest2.TestCase):
'level': 'level',
'devices': ['device_a', ],
'spares': ['device_b', ],
'keep_data': False,
}
new_md = partition.MD.from_dict(serialized)
assert serialized == new_md.to_dict()
@ -101,6 +102,7 @@ class TestPartition(unittest2.TestCase):
'guid': 'some_guid',
'name': 'name',
'partition_type': 'partition_type',
'keep_data': False,
}
new_pt = partition.Partition.from_dict(serialized)
assert serialized == new_pt.to_dict()
@ -161,6 +163,16 @@ class TestPartitionScheme(unittest2.TestCase):
self.assertRaises(errors.MDAlreadyExistsError,
self.p_scheme.md_next_name)
def test_partition_by_name(self):
parted_1 = self.p_scheme.add_parted(name='name_1', label='label_1')
parted_1.add_partition(size=1)
parted_2 = self.p_scheme.add_parted(name='name_2', label='label_2')
expected_prt = parted_2.add_partition(size=1)
actual_prt = self.p_scheme.partition_by_name(expected_prt.name)
self.assertEqual(expected_prt, actual_prt)
def test_md_by_name(self):
self.assertEqual(0, len(self.p_scheme.mds))
expected_md = partition.MD('name', 'level')
@ -204,6 +216,30 @@ class TestPartitionScheme(unittest2.TestCase):
self.assertEqual('fs_label', self.p_scheme.fss[0].label)
self.assertEqual('-F', self.p_scheme.fss[0].options)
def test_elevate_keep_data(self):
self.assertEqual(0, len(self.p_scheme.vgs))
self.assertEqual(0, len(self.p_scheme.lvs))
self.assertEqual(0, len(self.p_scheme.fss))
parted = self.p_scheme.add_parted(name='fake_name', label='fake_label')
prt = parted.add_partition(size=1, keep_data=True)
self.p_scheme.vg_attach_by_name(prt.name, 'fake_vg')
vg = self.p_scheme.vgs[0]
self.p_scheme.add_lv(name='fake_lv', vgname=vg.name, size=1)
lv = self.p_scheme.lvs[0]
self.p_scheme.add_fs(device=lv.device_name, mount='fake_mount',
fs_type='xfs', fs_label='fake_label')
fs = self.p_scheme.fss[0]
self.p_scheme.elevate_keep_data()
self.assertTrue(fs.keep_data)
self.assertFalse(lv.keep_data)
self.assertFalse(vg.keep_data)
self.assertFalse(prt.keep_data)
class TestParted(unittest2.TestCase):
def setUp(self):
@ -289,6 +325,14 @@ class TestParted(unittest2.TestCase):
self.prtd.partitions.extend(expected_partitions)
self.assertEqual(expected_partitions, self.prtd.primary)
def test_partition_by_name(self):
self.prtd.add_partition(size=1)
self.prtd.add_partition(size=1)
expected_prt = self.prtd.add_partition(size=1)
actual_prt = self.prtd.partition_by_name(expected_prt.name)
self.assertEqual(expected_prt, actual_prt)
def test_conversion(self):
prt = partition.Partition(
name='name',
@ -296,7 +340,8 @@ class TestParted(unittest2.TestCase):
device='device',
begin='begin',
end='end',
partition_type='primary'
partition_type='primary',
keep_data=True
)
self.prtd.partitions.append(prt)
serialized = self.prtd.to_dict()
@ -325,6 +370,7 @@ class TestLogicalVolume(unittest2.TestCase):
'name': 'lv-name',
'vgname': 'vg-name',
'size': 1234,
'keep_data': False,
}
new_lv = partition.LV.from_dict(serialized)
assert serialized == new_lv.to_dict()
@ -343,6 +389,7 @@ class TestPhisicalVolume(unittest2.TestCase):
'name': 'pv-name',
'metadatasize': 987,
'metadatacopies': 112,
'keep_data': False,
}
new_pv = partition.PV.from_dict(serialized)
assert serialized == new_pv.to_dict()
@ -358,7 +405,8 @@ class TestVolumesGroup(unittest2.TestCase):
serialized = vg.to_dict()
assert serialized == {
'name': 'vg-name',
'pvnames': ['pv-name-a', ]
'pvnames': ['pv-name-a', ],
'keep_data': False,
}
new_vg = partition.VG.from_dict(serialized)
assert serialized == new_vg.to_dict()
@ -381,6 +429,7 @@ class TestFileSystem(unittest2.TestCase):
'fs_type': 'type',
'fs_options': 'some-option',
'fs_label': 'some-label',
'keep_data': False,
}
new_fs = partition.FS.from_dict(serialized)
assert serialized == new_fs.to_dict()