diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py index 155e9f5a..ba5de40b 100644 --- a/fuel_agent/drivers/nailgun.py +++ b/fuel_agent/drivers/nailgun.py @@ -262,7 +262,9 @@ class Nailgun(BaseDataDriver): if volume.get('mount') != '/boot': LOG.debug('Adding partition on disk %s: size=%s' % (disk['name'], volume['size'])) - prt = parted.add_partition(size=volume['size']) + prt = parted.add_partition( + size=volume['size'], + keep_data=volume.get('keep_data', False)) LOG.debug('Partition name: %s' % prt.name) elif volume.get('mount') == '/boot' \ @@ -275,7 +277,9 @@ class Nailgun(BaseDataDriver): # huge disks if it is possible. LOG.debug('Adding /boot partition on disk %s: ' 'size=%s', disk['name'], volume['size']) - prt = parted.add_partition(size=volume['size']) + prt = parted.add_partition( + size=volume['size'], + keep_data=volume.get('keep_data', False)) LOG.debug('Partition name: %s', prt.name) self._boot_partition_done = True else: @@ -388,6 +392,7 @@ class Nailgun(BaseDataDriver): fs_type=volume.get('file_system', 'xfs'), fs_label=self._getlabel(volume.get('disk_label'))) + partition_scheme.elevate_keep_data() return partition_scheme def parse_configdrive_scheme(self): diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py index 9cfe1d54..2d810568 100644 --- a/fuel_agent/manager.py +++ b/fuel_agent/manager.py @@ -98,8 +98,27 @@ class Manager(object): def __init__(self, data): self.driver = utils.get_driver(CONF.data_driver)(data) + def do_clean_filesystems(self): + # All fs without keep_data flag and without image should be cleaned, + # mkfs will clean file table and such flags as --force are not needed + # TODO(asvechnikov): need to refactor processing keep_flag logic when + # data model will become flat + for fs in self.driver.partition_scheme.fss: + found_images = [img for img in self.driver.image_scheme.images + if img.target_device == fs.device] + + if not fs.keep_data and not found_images: + fu.make_fs(fs.type, fs.options, fs.label, fs.device) + def do_partitioning(self): LOG.debug('--- Partitioning disks (do_partitioning) ---') + + if self.driver.partition_scheme.skip_partitioning: + LOG.debug('Some of fs has keep_data flag, ' + 'partitioning is skiping') + self.do_clean_filesystems() + return + # If disks are not wiped out at all, it is likely they contain lvm # and md metadata which will prevent re-creating a partition table # with 'device is busy' error. diff --git a/fuel_agent/objects/base.py b/fuel_agent/objects/base.py index ff78d460..ea928156 100644 --- a/fuel_agent/objects/base.py +++ b/fuel_agent/objects/base.py @@ -17,8 +17,6 @@ import abc import six -from fuel_agent.utils.decorators import abstractclassmethod - @six.add_metaclass(abc.ABCMeta) class Serializable(object): @@ -27,6 +25,6 @@ class Serializable(object): def to_dict(self): pass - @abstractclassmethod + @classmethod def from_dict(cls, data): - pass + return cls(**data) diff --git a/fuel_agent/objects/partition.py b/fuel_agent/objects/partition.py index acd14677..d445d6c7 100644 --- a/fuel_agent/objects/partition.py +++ b/fuel_agent/objects/partition.py @@ -102,6 +102,11 @@ class Parted(base.Serializable): separator = 'p' return '%s%s%s' % (self.name, separator, self.next_count()) + def partition_by_name(self, name): + found = filter(lambda x: (x.name == name), self.partitions) + if found: + return found[0] + def to_dict(self): partitions = [partition.to_dict() for partition in self.partitions] return { @@ -123,7 +128,8 @@ class Parted(base.Serializable): class Partition(base.Serializable): def __init__(self, name, count, device, begin, end, partition_type, - flags=None, guid=None, configdrive=False): + flags=None, guid=None, configdrive=False, keep_data=False): + self.keep_data = keep_data self.name = name self.count = count self.device = device @@ -152,16 +158,15 @@ class Partition(base.Serializable): 'flags': self.flags, 'guid': self.guid, 'configdrive': self.configdrive, + 'keep_data': self.keep_data, } - @classmethod - def from_dict(cls, data): - return cls(**data) - class PhysicalVolume(base.Serializable): - def __init__(self, name, metadatasize=16, metadatacopies=2): + def __init__(self, name, metadatasize=16, + metadatacopies=2, keep_data=False): + self.keep_data = keep_data self.name = name self.metadatasize = metadatasize self.metadatacopies = metadatacopies @@ -171,19 +176,17 @@ class PhysicalVolume(base.Serializable): 'name': self.name, 'metadatasize': self.metadatasize, 'metadatacopies': self.metadatacopies, + 'keep_data': self.keep_data, } - @classmethod - def from_dict(cls, data): - return cls(**data) - PV = PhysicalVolume class VolumeGroup(base.Serializable): - def __init__(self, name, pvnames=None): + def __init__(self, name, pvnames=None, keep_data=False): + self.keep_data = keep_data self.name = name self.pvnames = pvnames or [] @@ -194,20 +197,18 @@ class VolumeGroup(base.Serializable): def to_dict(self): return { 'name': self.name, - 'pvnames': self.pvnames + 'pvnames': self.pvnames, + 'keep_data': self.keep_data, } - @classmethod - def from_dict(cls, data): - return cls(**data) - VG = VolumeGroup class LogicalVolume(base.Serializable): - def __init__(self, name, vgname, size): + def __init__(self, name, vgname, size, keep_data=False): + self.keep_data = keep_data self.name = name self.vgname = vgname self.size = size @@ -222,12 +223,9 @@ class LogicalVolume(base.Serializable): 'name': self.name, 'vgname': self.vgname, 'size': self.size, + 'keep_data': self.keep_data, } - @classmethod - def from_dict(cls, data): - return cls(**data) - LV = LogicalVolume @@ -235,7 +233,8 @@ LV = LogicalVolume class MultipleDevice(base.Serializable): def __init__(self, name, level, - devices=None, spares=None): + devices=None, spares=None, keep_data=False): + self.keep_data = keep_data self.name = name self.level = level self.devices = devices or [] @@ -261,20 +260,18 @@ class MultipleDevice(base.Serializable): 'level': self.level, 'devices': self.devices, 'spares': self.spares, + 'keep_data': self.keep_data, } - @classmethod - def from_dict(cls, data): - return cls(**data) - MD = MultipleDevice class FileSystem(base.Serializable): - def __init__(self, device, mount=None, - fs_type=None, fs_options=None, fs_label=None): + def __init__(self, device, mount=None, fs_type=None, + fs_options=None, fs_label=None, keep_data=False): + self.keep_data = keep_data self.device = device self.mount = mount self.type = fs_type or 'xfs' @@ -288,12 +285,9 @@ class FileSystem(base.Serializable): 'fs_type': self.type, 'fs_options': self.options, 'fs_label': self.label, + 'keep_data': self.keep_data, } - @classmethod - def from_dict(cls, data): - return cls(**data) - FS = FileSystem @@ -377,6 +371,11 @@ class PartitionScheme(object): 'try to generate md name manually') count += 1 + def partition_by_name(self, name): + return next((parted.partition_by_name(name) + for parted in self.parteds + if parted.partition_by_name(name)), None) + def vg_by_name(self, vgname): found = filter(lambda x: (x.name == vgname), self.vgs) if found: @@ -472,6 +471,56 @@ class PartitionScheme(object): if prt.configdrive: return prt.name + def elevate_keep_data(self): + LOG.debug('Elevate keep_data flag from partitions') + + for vg in self.vgs: + for pvname in vg.pvnames: + partition = self.partition_by_name(pvname) + if partition and partition.keep_data: + partition.keep_data = False + vg.keep_data = True + LOG.debug('Set keep_data to vg=%s' % vg.name) + + for lv in self.lvs: + vg = self.vg_by_name(lv.vgname) + if vg.keep_data: + lv.keep_data = True + + # Need to loop over lv again to remove keep flag from vg + for lv in self.lvs: + vg = self.vg_by_name(lv.vgname) + if vg.keep_data and lv.keep_data: + vg.keep_data = False + + for fs in self.fss: + lv = self.lv_by_device_name(fs.device) + if lv: + if lv.keep_data: + lv.keep_data = False + fs.keep_data = True + LOG.debug('Set keep_data to fs=%s from lv=%s' % + (fs.mount, lv.name)) + continue + partition = self.partition_by_name(fs.device) + if partition and partition.keep_data: + partition.keep_data = False + fs.keep_data = True + LOG.debug('Set keep flag to fs=%s from partition=%s' % + (fs.mount, partition.name)) + + @property + def skip_partitioning(self): + if any(fs.keep_data for fs in self.fss): + return True + if any(lv.keep_data for lv in self.lvs): + return True + if any(vg.keep_data for vg in self.vgs): + return True + for parted in self.parteds: + if any(prt.keep_data for prt in parted.partitions): + return True + def to_dict(self): return { 'parteds': [parted.to_dict() for parted in self.parteds], diff --git a/fuel_agent/tests/test_manager.py b/fuel_agent/tests/test_manager.py index c2beb77e..f1d20f38 100644 --- a/fuel_agent/tests/test_manager.py +++ b/fuel_agent/tests/test_manager.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import mock import os import signal @@ -146,6 +147,29 @@ class TestManager(test_base.BaseTestCase): mock_utils.makedirs_if_not_exists.assert_called_once_with( '/tmp/target/etc/nailgun-agent') + @mock.patch('yaml.load') + @mock.patch.object(utils, 'init_http_request') + @mock.patch.object(hu, 'list_block_devices') + @mock.patch.object(fu, 'make_fs') + def test_do_partitioning_with_keep_data_flag(self, mock_fu_mf, mock_lbd, + mock_http, mock_yaml): + mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE + data = copy.deepcopy(test_nailgun.PROVISION_SAMPLE_DATA) + + for disk in data['ks_meta']['pm_data']['ks_spaces']: + for volume in disk['volumes']: + if volume['type'] == 'pv' and volume['vg'] == 'image': + volume['keep_data'] = True + + self.mgr = manager.Manager(data) + + self.mgr.do_partitioning() + mock_fu_mf_expected_calls = [ + mock.call('ext2', '', '', '/dev/sda3'), + mock.call('ext2', '', '', '/dev/sda4'), + mock.call('swap', '', '', '/dev/mapper/os-swap')] + self.assertEqual(mock_fu_mf_expected_calls, mock_fu_mf.call_args_list) + @mock.patch('six.moves.builtins.open') @mock.patch.object(os, 'symlink') @mock.patch.object(os, 'remove') diff --git a/fuel_agent/tests/test_nailgun.py b/fuel_agent/tests/test_nailgun.py index 8fc59296..c067bc65 100644 --- a/fuel_agent/tests/test_nailgun.py +++ b/fuel_agent/tests/test_nailgun.py @@ -426,7 +426,8 @@ SINGLE_DISK_KS_SPACES = [ "size": 200, "type": "partition", "file_system": "ext4", - "name": "Root" + "name": "Root", + "keep_data": True }, ], "type": "disk", @@ -951,6 +952,37 @@ class TestNailgun(test_base.BaseTestCase): drv.partition_scheme.fs_by_mount('/boot').device, '/dev/sda3') + @mock.patch('fuel_agent.drivers.nailgun.yaml.load') + @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') + @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') + def test_elevate_keep_data_single_disk(self, mock_lbd, + mock_http_req, mock_yaml): + data = copy.deepcopy(PROVISION_SAMPLE_DATA) + data['ks_meta']['pm_data']['ks_spaces'] = SINGLE_DISK_KS_SPACES + mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE + drv = nailgun.Nailgun(data) + self.assertTrue(drv.partition_scheme.fs_by_mount('/').keep_data) + + for parted in drv.partition_scheme.parteds: + for partition in parted.partitions: + self.assertFalse(partition.keep_data) + + for md in drv.partition_scheme.mds: + self.assertFalse(md.keep_data) + + for pv in drv.partition_scheme.pvs: + self.assertFalse(pv.keep_data) + + for vg in drv.partition_scheme.vgs: + self.assertFalse(vg.keep_data) + + for lv in drv.partition_scheme.lvs: + self.assertFalse(lv.keep_data) + + for fs in drv.partition_scheme.fss: + if fs.mount != '/': + self.assertFalse(fs.keep_data) + @mock.patch('fuel_agent.drivers.nailgun.yaml.load') @mock.patch('fuel_agent.drivers.nailgun.utils.init_http_request') @mock.patch('fuel_agent.drivers.nailgun.hu.list_block_devices') diff --git a/fuel_agent/tests/test_partition.py b/fuel_agent/tests/test_partition.py index 72d9d3ae..cbc140dd 100644 --- a/fuel_agent/tests/test_partition.py +++ b/fuel_agent/tests/test_partition.py @@ -69,6 +69,7 @@ class TestMultipleDevice(unittest2.TestCase): 'level': 'level', 'devices': ['device_a', ], 'spares': ['device_b', ], + 'keep_data': False, } new_md = partition.MD.from_dict(serialized) assert serialized == new_md.to_dict() @@ -101,6 +102,7 @@ class TestPartition(unittest2.TestCase): 'guid': 'some_guid', 'name': 'name', 'partition_type': 'partition_type', + 'keep_data': False, } new_pt = partition.Partition.from_dict(serialized) assert serialized == new_pt.to_dict() @@ -161,6 +163,16 @@ class TestPartitionScheme(unittest2.TestCase): self.assertRaises(errors.MDAlreadyExistsError, self.p_scheme.md_next_name) + def test_partition_by_name(self): + parted_1 = self.p_scheme.add_parted(name='name_1', label='label_1') + parted_1.add_partition(size=1) + + parted_2 = self.p_scheme.add_parted(name='name_2', label='label_2') + expected_prt = parted_2.add_partition(size=1) + + actual_prt = self.p_scheme.partition_by_name(expected_prt.name) + self.assertEqual(expected_prt, actual_prt) + def test_md_by_name(self): self.assertEqual(0, len(self.p_scheme.mds)) expected_md = partition.MD('name', 'level') @@ -204,6 +216,30 @@ class TestPartitionScheme(unittest2.TestCase): self.assertEqual('fs_label', self.p_scheme.fss[0].label) self.assertEqual('-F', self.p_scheme.fss[0].options) + def test_elevate_keep_data(self): + self.assertEqual(0, len(self.p_scheme.vgs)) + self.assertEqual(0, len(self.p_scheme.lvs)) + self.assertEqual(0, len(self.p_scheme.fss)) + parted = self.p_scheme.add_parted(name='fake_name', label='fake_label') + prt = parted.add_partition(size=1, keep_data=True) + + self.p_scheme.vg_attach_by_name(prt.name, 'fake_vg') + vg = self.p_scheme.vgs[0] + + self.p_scheme.add_lv(name='fake_lv', vgname=vg.name, size=1) + lv = self.p_scheme.lvs[0] + + self.p_scheme.add_fs(device=lv.device_name, mount='fake_mount', + fs_type='xfs', fs_label='fake_label') + fs = self.p_scheme.fss[0] + + self.p_scheme.elevate_keep_data() + + self.assertTrue(fs.keep_data) + self.assertFalse(lv.keep_data) + self.assertFalse(vg.keep_data) + self.assertFalse(prt.keep_data) + class TestParted(unittest2.TestCase): def setUp(self): @@ -289,6 +325,14 @@ class TestParted(unittest2.TestCase): self.prtd.partitions.extend(expected_partitions) self.assertEqual(expected_partitions, self.prtd.primary) + def test_partition_by_name(self): + self.prtd.add_partition(size=1) + self.prtd.add_partition(size=1) + expected_prt = self.prtd.add_partition(size=1) + + actual_prt = self.prtd.partition_by_name(expected_prt.name) + self.assertEqual(expected_prt, actual_prt) + def test_conversion(self): prt = partition.Partition( name='name', @@ -296,7 +340,8 @@ class TestParted(unittest2.TestCase): device='device', begin='begin', end='end', - partition_type='primary' + partition_type='primary', + keep_data=True ) self.prtd.partitions.append(prt) serialized = self.prtd.to_dict() @@ -325,6 +370,7 @@ class TestLogicalVolume(unittest2.TestCase): 'name': 'lv-name', 'vgname': 'vg-name', 'size': 1234, + 'keep_data': False, } new_lv = partition.LV.from_dict(serialized) assert serialized == new_lv.to_dict() @@ -343,6 +389,7 @@ class TestPhisicalVolume(unittest2.TestCase): 'name': 'pv-name', 'metadatasize': 987, 'metadatacopies': 112, + 'keep_data': False, } new_pv = partition.PV.from_dict(serialized) assert serialized == new_pv.to_dict() @@ -358,7 +405,8 @@ class TestVolumesGroup(unittest2.TestCase): serialized = vg.to_dict() assert serialized == { 'name': 'vg-name', - 'pvnames': ['pv-name-a', ] + 'pvnames': ['pv-name-a', ], + 'keep_data': False, } new_vg = partition.VG.from_dict(serialized) assert serialized == new_vg.to_dict() @@ -381,6 +429,7 @@ class TestFileSystem(unittest2.TestCase): 'fs_type': 'type', 'fs_options': 'some-option', 'fs_label': 'some-label', + 'keep_data': False, } new_fs = partition.FS.from_dict(serialized) assert serialized == new_fs.to_dict()