Move files from mount points to actual file systems
The thing is that sometimes we have file system images and mount point hierachies which are not aligned. Let's say, we have root file system image, while partition scheme says that two file systems should be created on the node: / and /var. In this case root image has /var directory with a set of files. Obviously, we need to move all these files from /var directory on the root file system to /var file system because /var directory will be used as mount point. In order to achieve this we mount all existent file systems into a flat set of temporary directories. We then try to find specific paths which correspond to mount points and move all files from these paths to corresponding file systems. Co-Authored-By: Alexey Stupnikov <astupnikov@mirantis.com> Change-Id: I1aa7523055ac4bcf6f8a93e9740ccf652ed35cc1 Closes-Bug: #1537699
This commit is contained in:
Vladimir Kozhukalov
Alexey Stupnikov
parent
ebff0b0ba1
commit
5809a3fea9
|
|
@ -475,6 +475,85 @@ class Manager(object):
|
|||
LOG.debug('Extending %s %s' %
|
||||
(image.format, image.target_device))
|
||||
fu.extend_fs(image.format, image.target_device)
|
||||
self.move_files_to_their_places()
|
||||
|
||||
def move_files_to_their_places(self, remove_src=True):
|
||||
"""Move files from mount points to where those files should be.
|
||||
|
||||
:param remove_src: Remove source files after sync if True (default).
|
||||
"""
|
||||
|
||||
# NOTE(kozhukalov): The thing is that sometimes we
|
||||
# have file system images and mount point hierachies
|
||||
# which are not aligned. Let's say, we have root file system
|
||||
# image, while partition scheme says that two file systems should
|
||||
# be created on the node: / and /var.
|
||||
# In this case root image has /var directory with a set of files.
|
||||
# Obviously, we need to move all these files from /var directory
|
||||
# on the root file system to /var file system because /var
|
||||
# directory will be used as mount point.
|
||||
# In order to achieve this we mount all existent file
|
||||
# systems into a flat set of temporary directories. We then
|
||||
# try to find specific paths which correspond to mount points
|
||||
# and move all files from these paths to corresponding file systems.
|
||||
|
||||
mount_map = self.mount_target_flat()
|
||||
for fs_mount in sorted(mount_map):
|
||||
head, tail = os.path.split(fs_mount)
|
||||
while head != fs_mount:
|
||||
LOG.debug('Trying to move files for %s file system', fs_mount)
|
||||
if head in mount_map:
|
||||
LOG.debug('File system %s is mounted into %s',
|
||||
head, mount_map[head])
|
||||
check_path = os.path.join(mount_map[head], tail)
|
||||
LOG.debug('Trying to check if path %s exists', check_path)
|
||||
if os.path.exists(check_path):
|
||||
LOG.debug('Path %s exists. Trying to sync all files '
|
||||
'from there to %s', mount_map[fs_mount])
|
||||
src_path = check_path + '/'
|
||||
utils.execute('rsync', '-avH', src_path,
|
||||
mount_map[fs_mount])
|
||||
if remove_src:
|
||||
shutil.rmtree(check_path)
|
||||
break
|
||||
if head == '/':
|
||||
break
|
||||
head, _tail = os.path.split(head)
|
||||
tail = os.path.join(_tail, tail)
|
||||
self.umount_target_flat(mount_map)
|
||||
|
||||
def mount_target_flat(self):
|
||||
"""Mount a set of file systems into a set of temporary directories
|
||||
|
||||
:returns: Mount map dict
|
||||
"""
|
||||
|
||||
LOG.debug('Mounting target file systems into a flat set '
|
||||
'of temporary directories')
|
||||
mount_map = {}
|
||||
for fs in self.driver.partition_scheme.fss:
|
||||
if fs.mount == 'swap':
|
||||
continue
|
||||
# It is an ugly hack to resolve python2/3 encoding issues and
|
||||
# should be removed after transistion to python3
|
||||
try:
|
||||
type(fs.mount) is unicode
|
||||
fs_mount = fs.mount.encode('ascii', 'ignore')
|
||||
except NameError:
|
||||
fs_mount = fs.mount
|
||||
mount_map[fs_mount] = fu.mount_fs_temp(fs.type, str(fs.device))
|
||||
LOG.debug('Flat mount map: %s', mount_map)
|
||||
return mount_map
|
||||
|
||||
def umount_target_flat(self, mount_map):
|
||||
"""Umount file systems previously mounted into temporary directories.
|
||||
|
||||
:param mount_map: Mount map dict
|
||||
"""
|
||||
|
||||
for mount_point in six.itervalues(mount_map):
|
||||
fu.umount_fs(mount_point)
|
||||
shutil.rmtree(mount_point)
|
||||
|
||||
@staticmethod
|
||||
def _update_metadata_with_repos(metadata, repos):
|
||||
|
|
@ -866,20 +945,6 @@ class Manager(object):
|
|||
with open(os.path.join(chroot, 'etc/nailgun-agent/nodiscover'), 'w'):
|
||||
pass
|
||||
|
||||
# FIXME(kozhukalov): When we have just os-root fs image and don't have
|
||||
# os-var-log fs image while / and /var/log are supposed to be
|
||||
# separate file systems and os-var-log is mounted into
|
||||
# non-empty directory on the / file system, those files in /var/log
|
||||
# directory become unavailable.
|
||||
# The thing is that among those file there is /var/log/upstart
|
||||
# where upstart daemon writes its logs. We have specific upstart job
|
||||
# which is to flush open files once all file systems are mounted.
|
||||
# This job needs upstart directory to be available on os-var-log
|
||||
# file system.
|
||||
# This is just a temporary fix and correct fix will be available soon
|
||||
# via updates.
|
||||
utils.execute('mkdir', '-p', chroot + '/var/log/upstart')
|
||||
|
||||
with open(chroot + '/etc/fstab', 'wt', encoding='utf-8') as f:
|
||||
for fs in self.driver.partition_scheme.fss:
|
||||
# TODO(kozhukalov): Think of improving the logic so as to
|
||||
|
|
|
|||
|
|
@ -170,6 +170,14 @@ class TestFSUtils(unittest2.TestCase):
|
|||
'/dev/sda4')
|
||||
self.assertEqual(ret, 'megafs')
|
||||
|
||||
@mock.patch('fuel_agent.utils.fs.mount_fs')
|
||||
@mock.patch('fuel_agent.utils.fs.tempfile.mkdtemp')
|
||||
def test_mount_fs_temp(self, mock_mkdtemp, mock_mount, mock_exec):
|
||||
mock_mkdtemp.return_value = '/tmp/dir'
|
||||
self.assertEqual('/tmp/dir', fu.mount_fs_temp('ext4', '/dev/fake'))
|
||||
mock_mkdtemp.assert_called_once_with(dir=None, suffix='')
|
||||
mock_mount.assert_called_once_with('ext4', '/dev/fake', '/tmp/dir')
|
||||
|
||||
|
||||
class TestFSRetry(unittest2.TestCase):
|
||||
|
||||
|
|
|
|||
|
|
@ -619,6 +619,7 @@ class TestManager(unittest2.TestCase):
|
|||
mocks['_make_configdrive_image'].assert_called_once_with('x')
|
||||
mocks['_add_configdrive_image'].assert_called_once_with()
|
||||
|
||||
@mock.patch('fuel_agent.manager.Manager.move_files_to_their_places')
|
||||
@mock.patch.object(fu, 'get_fs_type')
|
||||
@mock.patch.object(manager.os.path, 'exists')
|
||||
@mock.patch.object(hu, 'is_block_device')
|
||||
|
|
@ -637,7 +638,8 @@ class TestManager(unittest2.TestCase):
|
|||
def test_do_copyimage(self, mock_lbd, mock_u_ras, mock_u_e, mock_au_c,
|
||||
mock_au_h, mock_au_l, mock_au_g, mock_fu_ef,
|
||||
mock_http_req, mock_yaml, mock_get_size, mock_md5,
|
||||
mock_ibd, mock_os_path, mock_get_fs_type):
|
||||
mock_ibd, mock_os_path, mock_get_fs_type,
|
||||
mock_mfttp):
|
||||
mock_os_path.return_value = True
|
||||
mock_ibd.return_value = True
|
||||
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
|
||||
|
|
@ -664,6 +666,7 @@ class TestManager(unittest2.TestCase):
|
|||
mock_fu_ef_expected_calls = [
|
||||
mock.call('ext4', '/dev/mapper/os-root')]
|
||||
self.assertEqual(mock_fu_ef_expected_calls, mock_fu_ef.call_args_list)
|
||||
self.assertTrue(mock_mfttp.called)
|
||||
|
||||
@mock.patch.object(fu, 'get_fs_type')
|
||||
@mock.patch.object(manager.os.path, 'exists')
|
||||
|
|
@ -728,6 +731,7 @@ class TestManager(unittest2.TestCase):
|
|||
with self.assertRaisesRegexp(errors.WrongDeviceError, msg):
|
||||
self.mgr.do_copyimage()
|
||||
|
||||
@mock.patch('fuel_agent.manager.Manager.move_files_to_their_places')
|
||||
@mock.patch.object(fu, 'get_fs_type')
|
||||
@mock.patch.object(manager.os.path, 'exists')
|
||||
@mock.patch.object(hu, 'is_block_device')
|
||||
|
|
@ -748,7 +752,7 @@ class TestManager(unittest2.TestCase):
|
|||
mock_au_g, mock_fu_ef, mock_http_req,
|
||||
mock_yaml, mock_get_size, mock_md5,
|
||||
mock_ibd, mock_os_path,
|
||||
mock_get_fs_type):
|
||||
mock_get_fs_type, mock_mfttp):
|
||||
mock_os_path.return_value = True
|
||||
mock_ibd.return_value = True
|
||||
mock_get_size.return_value = 123
|
||||
|
|
@ -764,6 +768,7 @@ class TestManager(unittest2.TestCase):
|
|||
mock.call('/dev/mapper/os-root', 1234),
|
||||
mock.call('/dev/sda7', 123)]
|
||||
self.assertEqual(expected_md5_calls, mock_md5.call_args_list)
|
||||
self.assertTrue(mock_mfttp.called)
|
||||
|
||||
@mock.patch.object(fu, 'get_fs_type')
|
||||
@mock.patch.object(hu, 'is_block_device')
|
||||
|
|
@ -894,6 +899,77 @@ none /run/shm tmpfs rw,nosuid,nodev 0 0"""
|
|||
mock.call('fake_chroot/')],
|
||||
mock_fu.umount_fs.call_args_list)
|
||||
|
||||
@mock.patch('fuel_agent.utils.fs.mount_fs_temp')
|
||||
def test_mount_target_flat(self, mock_mfst):
|
||||
def mfst_side_effect(*args, **kwargs):
|
||||
if '/dev/fake1' in args:
|
||||
return '/tmp/dir1'
|
||||
elif '/dev/fake2' in args:
|
||||
return '/tmp/dir2'
|
||||
mock_mfst.side_effect = mfst_side_effect
|
||||
self.mgr.driver._partition_scheme = objects.PartitionScheme()
|
||||
self.mgr.driver.partition_scheme.add_fs(
|
||||
device='/dev/fake1', mount='/', fs_type='ext4')
|
||||
self.mgr.driver.partition_scheme.add_fs(
|
||||
device='/dev/fake2', mount='/var/lib', fs_type='ext4')
|
||||
self.assertEqual({'/': '/tmp/dir1', '/var/lib': '/tmp/dir2'},
|
||||
self.mgr.mount_target_flat())
|
||||
self.assertEqual([mock.call('ext4', '/dev/fake1'),
|
||||
mock.call('ext4', '/dev/fake2')],
|
||||
mock_mfst.call_args_list)
|
||||
|
||||
@mock.patch('fuel_agent.manager.shutil.rmtree')
|
||||
@mock.patch('fuel_agent.utils.fs.umount_fs')
|
||||
def test_umount_target_flat(self, mock_umfs, mock_rmtree):
|
||||
mount_map = {'/': '/tmp/dir1', '/var/lib': '/tmp/dir2'}
|
||||
self.mgr.umount_target_flat(mount_map)
|
||||
mock_umfs.assert_has_calls(
|
||||
[mock.call('/tmp/dir1'), mock.call('/tmp/dir2')],
|
||||
any_order=True)
|
||||
|
||||
@mock.patch('fuel_agent.manager.shutil.rmtree')
|
||||
@mock.patch('fuel_agent.manager.os.path.exists')
|
||||
@mock.patch('fuel_agent.manager.utils.execute')
|
||||
@mock.patch('fuel_agent.manager.Manager.umount_target_flat')
|
||||
@mock.patch('fuel_agent.manager.Manager.mount_target_flat')
|
||||
def test_move_files_to_their_places(self, mock_mtf, mock_utf,
|
||||
mock_ute, mock_ope, mock_shrmt):
|
||||
|
||||
def ope_side_effect(path):
|
||||
if path == '/tmp/dir1/var/lib':
|
||||
return True
|
||||
|
||||
mock_ope.side_effect = ope_side_effect
|
||||
mock_mtf.return_value = {'/': '/tmp/dir1', '/var/lib': '/tmp/dir2'}
|
||||
self.mgr.move_files_to_their_places()
|
||||
self.assertEqual(
|
||||
[mock.call('rsync', '-avH', '/tmp/dir1/var/lib/', '/tmp/dir2')],
|
||||
mock_ute.call_args_list)
|
||||
self.assertEqual(
|
||||
[mock.call('/tmp/dir1/var/lib')],
|
||||
mock_shrmt.call_args_list)
|
||||
|
||||
@mock.patch('fuel_agent.manager.shutil.rmtree')
|
||||
@mock.patch('fuel_agent.manager.os.path.exists')
|
||||
@mock.patch('fuel_agent.manager.utils.execute')
|
||||
@mock.patch('fuel_agent.manager.Manager.umount_target_flat')
|
||||
@mock.patch('fuel_agent.manager.Manager.mount_target_flat')
|
||||
def test_move_files_to_their_places_not_remove(self, mock_mtf, mock_utf,
|
||||
mock_ute, mock_ope,
|
||||
mock_shrmt):
|
||||
|
||||
def ope_side_effect(path):
|
||||
if path == '/tmp/dir1/var/lib':
|
||||
return True
|
||||
|
||||
mock_ope.side_effect = ope_side_effect
|
||||
mock_mtf.return_value = {'/': '/tmp/dir1', '/var/lib': '/tmp/dir2'}
|
||||
self.mgr.move_files_to_their_places(remove_src=False)
|
||||
self.assertEqual(
|
||||
[mock.call('rsync', '-avH', '/tmp/dir1/var/lib/', '/tmp/dir2')],
|
||||
mock_ute.call_args_list)
|
||||
self.assertFalse(mock_shrmt.called)
|
||||
|
||||
|
||||
class TestImageBuild(unittest2.TestCase):
|
||||
@mock.patch('yaml.load')
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import tempfile
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
from fuel_agent.utils import utils
|
||||
|
|
@ -124,3 +126,9 @@ def get_fs_type(device):
|
|||
output = utils.execute('blkid', '-o', 'value', '-s', 'TYPE',
|
||||
'-c', '/dev/null', device)[0]
|
||||
return output.strip()
|
||||
|
||||
|
||||
def mount_fs_temp(fs_type, fs_dev, tmpdir=None, suffix=''):
|
||||
mount_point = tempfile.mkdtemp(dir=tmpdir, suffix=suffix)
|
||||
mount_fs(fs_type, fs_dev, mount_point)
|
||||
return mount_point
|
||||
|
|
|
|||
Loading…
Reference in New Issue