IBP: add cleanup for image building
This patch wraps image build procedure into try-except-finally and appends some cleaning actions into finally section. If something goes wrong during image building, build environment will be cleaned up. 1) Stop running processes inside image chroot 2) Umount image chroot 3) Detach loop devices 4) Remove temporary files Closes-Bug: #1451817 Partially implements: blueprint ibp-build-ubuntu-images Change-Id: I7a7abf46f93fad3a3343dcf320781beec42f44bd
This commit is contained in:
Alexander Gordeev
Vladimir Kozhukalov
parent
c936d8a794
commit
6dffe4c5b3
|
|
@ -158,3 +158,7 @@ class WrongRepositoryError(BaseError):
|
|||
|
||||
class WrongDeviceError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class UnexpectedProcessError(BaseError):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ import os
|
|||
import shutil
|
||||
import signal
|
||||
import tempfile
|
||||
import time
|
||||
import yaml
|
||||
|
||||
from oslo.config import cfg
|
||||
|
|
@ -342,7 +341,7 @@ class Manager(object):
|
|||
:param pseudo: If pseudo file systems
|
||||
need to be mounted (Default: True)
|
||||
"""
|
||||
LOG.debug('Mounting target file systems')
|
||||
LOG.debug('Mounting target file systems: %s', chroot)
|
||||
# Here we are going to mount all file systems in partition scheme.
|
||||
for fs in self.driver.partition_scheme.fs_sorted_by_depth():
|
||||
if fs.mount == 'swap':
|
||||
|
|
@ -366,16 +365,16 @@ class Manager(object):
|
|||
f.write(mtab)
|
||||
|
||||
# TODO(kozhukalov): write tests for this method
|
||||
def umount_target(self, chroot, pseudo=True):
|
||||
LOG.debug('Umounting target file systems')
|
||||
def umount_target(self, chroot, pseudo=True, try_lazy_umount=True):
|
||||
LOG.debug('Umounting target file systems: %s', chroot)
|
||||
if pseudo:
|
||||
for path in ('/proc', '/dev', '/sys'):
|
||||
fu.umount_fs(chroot + path)
|
||||
fu.umount_fs(chroot + path, try_lazy_umount=try_lazy_umount)
|
||||
for fs in self.driver.partition_scheme.fs_sorted_by_depth(
|
||||
reverse=True):
|
||||
if fs.mount == 'swap':
|
||||
continue
|
||||
fu.umount_fs(chroot + fs.mount)
|
||||
fu.umount_fs(chroot + fs.mount, try_lazy_umount=try_lazy_umount)
|
||||
|
||||
# TODO(kozhukalov): write tests for this method
|
||||
# https://bugs.launchpad.net/fuel/+bug/1449609
|
||||
|
|
@ -492,167 +491,205 @@ class Manager(object):
|
|||
return
|
||||
LOG.debug('At least one of the necessary images is unavailable. '
|
||||
'Starting build process.')
|
||||
|
||||
LOG.info('*** Preparing image space ***')
|
||||
for image in self.driver.image_scheme.images:
|
||||
LOG.debug('Creating temporary sparsed file for the '
|
||||
'image: %s', image.uri)
|
||||
img_tmp_file = bu.create_sparse_tmp_file(
|
||||
try:
|
||||
LOG.debug('Creating temporary chroot directory')
|
||||
chroot = tempfile.mkdtemp(
|
||||
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
|
||||
LOG.debug('Temporary file: %s', img_tmp_file)
|
||||
LOG.debug('Temporary chroot: %s', chroot)
|
||||
|
||||
# we need to remember those files
|
||||
# to be able to shrink them and move in the end
|
||||
image.img_tmp_file = img_tmp_file
|
||||
proc_path = os.path.join(chroot, 'proc')
|
||||
|
||||
LOG.debug('Looking for a free loop device')
|
||||
image.target_device.name = bu.get_free_loop_device()
|
||||
LOG.info('*** Preparing image space ***')
|
||||
for image in self.driver.image_scheme.images:
|
||||
LOG.debug('Creating temporary sparsed file for the '
|
||||
'image: %s', image.uri)
|
||||
img_tmp_file = bu.create_sparse_tmp_file(
|
||||
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
|
||||
LOG.debug('Temporary file: %s', img_tmp_file)
|
||||
|
||||
LOG.debug('Attaching temporary image file to free loop device')
|
||||
bu.attach_file_to_loop(img_tmp_file, str(image.target_device))
|
||||
# we need to remember those files
|
||||
# to be able to shrink them and move in the end
|
||||
image.img_tmp_file = img_tmp_file
|
||||
|
||||
# find fs with the same loop device object
|
||||
# as image.target_device
|
||||
fs = self.driver.partition_scheme.fs_by_device(image.target_device)
|
||||
LOG.debug('Looking for a free loop device')
|
||||
image.target_device.name = bu.get_free_loop_device()
|
||||
|
||||
LOG.debug('Creating file system on the image')
|
||||
fu.make_fs(
|
||||
fs_type=fs.type,
|
||||
fs_options=fs.options,
|
||||
fs_label=fs.label,
|
||||
dev=str(fs.device))
|
||||
LOG.debug('Attaching temporary image file to free loop device')
|
||||
bu.attach_file_to_loop(img_tmp_file, str(image.target_device))
|
||||
|
||||
LOG.debug('Creating temporary chroot directory')
|
||||
chroot = tempfile.mkdtemp(
|
||||
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
|
||||
LOG.debug('Temporary chroot: %s', chroot)
|
||||
# find fs with the same loop device object
|
||||
# as image.target_device
|
||||
fs = self.driver.partition_scheme.fs_by_device(
|
||||
image.target_device)
|
||||
|
||||
# mounting all images into chroot tree
|
||||
self.mount_target(chroot, treat_mtab=False, pseudo=False)
|
||||
LOG.debug('Creating file system on the image')
|
||||
fu.make_fs(
|
||||
fs_type=fs.type,
|
||||
fs_options=fs.options,
|
||||
fs_label=fs.label,
|
||||
dev=str(fs.device))
|
||||
|
||||
LOG.info('*** Shipping image content ***')
|
||||
LOG.debug('Installing operating system into image')
|
||||
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
|
||||
# mounting all images into chroot tree
|
||||
self.mount_target(chroot, treat_mtab=False, pseudo=False)
|
||||
|
||||
# DEBOOTSTRAP
|
||||
# we use first repo as the main mirror
|
||||
uri = self.driver.operating_system.repos[0].uri
|
||||
suite = self.driver.operating_system.repos[0].suite
|
||||
LOG.info('*** Shipping image content ***')
|
||||
LOG.debug('Installing operating system into image')
|
||||
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
|
||||
|
||||
LOG.debug('Preventing services from being get started')
|
||||
bu.suppress_services_start(chroot)
|
||||
LOG.debug('Installing base operating system using debootstrap')
|
||||
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot)
|
||||
# DEBOOTSTRAP
|
||||
# we use first repo as the main mirror
|
||||
uri = self.driver.operating_system.repos[0].uri
|
||||
suite = self.driver.operating_system.repos[0].suite
|
||||
|
||||
# APT-GET
|
||||
LOG.debug('Configuring apt inside chroot')
|
||||
LOG.debug('Setting environment variables')
|
||||
bu.set_apt_get_env()
|
||||
LOG.debug('Allowing unauthenticated repos')
|
||||
bu.pre_apt_get(chroot)
|
||||
LOG.debug('Preventing services from being get started')
|
||||
bu.suppress_services_start(chroot)
|
||||
LOG.debug('Installing base operating system using debootstrap')
|
||||
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot)
|
||||
|
||||
for repo in self.driver.operating_system.repos:
|
||||
LOG.debug('Adding repository source: name={name}, uri={uri},'
|
||||
'suite={suite}, section={section}'.format(
|
||||
name=repo.name, uri=repo.uri,
|
||||
suite=repo.suite, section=repo.section))
|
||||
bu.add_apt_source(
|
||||
name=repo.name,
|
||||
uri=repo.uri,
|
||||
suite=repo.suite,
|
||||
section=repo.section,
|
||||
chroot=chroot)
|
||||
LOG.debug('Adding repository preference: '
|
||||
'name={name}, priority={priority}'.format(
|
||||
name=repo.name, priority=repo.priority))
|
||||
if repo.priority is not None:
|
||||
bu.add_apt_preference(
|
||||
# APT-GET
|
||||
LOG.debug('Configuring apt inside chroot')
|
||||
LOG.debug('Setting environment variables')
|
||||
bu.set_apt_get_env()
|
||||
LOG.debug('Allowing unauthenticated repos')
|
||||
bu.pre_apt_get(chroot)
|
||||
|
||||
for repo in self.driver.operating_system.repos:
|
||||
LOG.debug('Adding repository source: name={name}, uri={uri},'
|
||||
'suite={suite}, section={section}'.format(
|
||||
name=repo.name, uri=repo.uri,
|
||||
suite=repo.suite, section=repo.section))
|
||||
bu.add_apt_source(
|
||||
name=repo.name,
|
||||
priority=repo.priority,
|
||||
uri=repo.uri,
|
||||
suite=repo.suite,
|
||||
section=repo.section,
|
||||
chroot=chroot,
|
||||
uri=repo.uri)
|
||||
chroot=chroot)
|
||||
LOG.debug('Adding repository preference: '
|
||||
'name={name}, priority={priority}'.format(
|
||||
name=repo.name, priority=repo.priority))
|
||||
if repo.priority is not None:
|
||||
bu.add_apt_preference(
|
||||
name=repo.name,
|
||||
priority=repo.priority,
|
||||
suite=repo.suite,
|
||||
section=repo.section,
|
||||
chroot=chroot,
|
||||
uri=repo.uri)
|
||||
|
||||
metadata.setdefault('repos', []).append({
|
||||
'type': 'deb',
|
||||
'name': repo.name,
|
||||
'uri': repo.uri,
|
||||
'suite': repo.suite,
|
||||
'section': repo.section,
|
||||
'priority': repo.priority,
|
||||
'meta': repo.meta})
|
||||
metadata.setdefault('repos', []).append({
|
||||
'type': 'deb',
|
||||
'name': repo.name,
|
||||
'uri': repo.uri,
|
||||
'suite': repo.suite,
|
||||
'section': repo.section,
|
||||
'priority': repo.priority,
|
||||
'meta': repo.meta})
|
||||
|
||||
LOG.debug('Preventing services from being get started')
|
||||
bu.suppress_services_start(chroot)
|
||||
LOG.debug('Preventing services from being get started')
|
||||
bu.suppress_services_start(chroot)
|
||||
|
||||
packages = self.driver.operating_system.packages
|
||||
metadata['packages'] = packages
|
||||
packages = self.driver.operating_system.packages
|
||||
metadata['packages'] = packages
|
||||
|
||||
# we need /proc to be mounted for apt-get success
|
||||
proc_path = os.path.join(chroot, 'proc')
|
||||
utils.makedirs_if_not_exists(proc_path)
|
||||
fu.mount_bind(chroot, '/proc')
|
||||
# we need /proc to be mounted for apt-get success
|
||||
utils.makedirs_if_not_exists(proc_path)
|
||||
fu.mount_bind(chroot, '/proc')
|
||||
|
||||
LOG.debug('Installing packages using apt-get: %s',
|
||||
' '.join(packages))
|
||||
bu.run_apt_get(chroot, packages=packages)
|
||||
LOG.debug('Installing packages using apt-get: %s',
|
||||
' '.join(packages))
|
||||
bu.run_apt_get(chroot, packages=packages)
|
||||
|
||||
LOG.debug('Post-install OS configuration')
|
||||
bu.do_post_inst(chroot)
|
||||
LOG.debug('Post-install OS configuration')
|
||||
bu.do_post_inst(chroot)
|
||||
|
||||
LOG.debug('Making sure there are no running processes '
|
||||
'inside chroot before trying to umount chroot')
|
||||
bu.send_signal_to_chrooted_processes(chroot, signal.SIGTERM)
|
||||
# We assume there might be some processes which
|
||||
# require some reasonable time to stop before we try
|
||||
# to send them SIGKILL. Waiting for 2 seconds
|
||||
# looks reasonable here.
|
||||
time.sleep(2)
|
||||
bu.send_signal_to_chrooted_processes(chroot, signal.SIGKILL)
|
||||
LOG.debug('Making sure there are no running processes '
|
||||
'inside chroot before trying to umount chroot')
|
||||
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
|
||||
if not bu.stop_chrooted_processes(
|
||||
chroot, signal=signal.SIGKILL):
|
||||
raise errors.UnexpectedProcessError(
|
||||
'Stopping chrooted processes failed. '
|
||||
'There are some processes running in chroot %s',
|
||||
chroot)
|
||||
|
||||
LOG.info('*** Finalizing image space ***')
|
||||
fu.umount_fs(proc_path)
|
||||
# umounting all loop devices
|
||||
self.umount_target(chroot, pseudo=False)
|
||||
LOG.info('*** Finalizing image space ***')
|
||||
fu.umount_fs(proc_path)
|
||||
# umounting all loop devices
|
||||
self.umount_target(chroot, pseudo=False, try_lazy_umount=False)
|
||||
|
||||
for image in self.driver.image_scheme.images:
|
||||
LOG.debug('Deattaching loop device from file: %s',
|
||||
image.img_tmp_file)
|
||||
bu.deattach_loop(str(image.target_device))
|
||||
LOG.debug('Shrinking temporary image file: %s',
|
||||
image.img_tmp_file)
|
||||
bu.shrink_sparse_file(image.img_tmp_file)
|
||||
for image in self.driver.image_scheme.images:
|
||||
LOG.debug('Deattaching loop device from file: %s',
|
||||
image.img_tmp_file)
|
||||
bu.deattach_loop(str(image.target_device))
|
||||
LOG.debug('Shrinking temporary image file: %s',
|
||||
image.img_tmp_file)
|
||||
bu.shrink_sparse_file(image.img_tmp_file)
|
||||
|
||||
raw_size = os.path.getsize(image.img_tmp_file)
|
||||
raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)
|
||||
raw_size = os.path.getsize(image.img_tmp_file)
|
||||
raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)
|
||||
|
||||
LOG.debug('Containerizing temporary image file: %s',
|
||||
image.img_tmp_file)
|
||||
img_tmp_containerized = bu.containerize(
|
||||
image.img_tmp_file, image.container)
|
||||
img_containerized = image.uri.split('file://', 1)[1]
|
||||
LOG.debug('Containerizing temporary image file: %s',
|
||||
image.img_tmp_file)
|
||||
img_tmp_containerized = bu.containerize(
|
||||
image.img_tmp_file, image.container)
|
||||
img_containerized = image.uri.split('file://', 1)[1]
|
||||
|
||||
# NOTE(kozhukalov): implement abstract publisher
|
||||
LOG.debug('Moving image file to the final location: %s',
|
||||
img_containerized)
|
||||
shutil.move(img_tmp_containerized, img_containerized)
|
||||
|
||||
container_size = os.path.getsize(img_containerized)
|
||||
container_md5 = utils.calculate_md5(
|
||||
img_containerized, container_size)
|
||||
metadata.setdefault('images', []).append({
|
||||
'raw_md5': raw_md5,
|
||||
'raw_size': raw_size,
|
||||
'raw_name': None,
|
||||
'container_name': os.path.basename(img_containerized),
|
||||
'container_md5': container_md5,
|
||||
'container_size': container_size,
|
||||
'container': image.container,
|
||||
'format': image.format})
|
||||
|
||||
# NOTE(kozhukalov): implement abstract publisher
|
||||
LOG.debug('Moving image file to the final location: %s',
|
||||
img_containerized)
|
||||
shutil.move(img_tmp_containerized, img_containerized)
|
||||
LOG.debug('Image metadata: %s', metadata)
|
||||
with open(self.driver.metadata_uri.split('file://', 1)[1],
|
||||
'w') as f:
|
||||
yaml.safe_dump(metadata, stream=f)
|
||||
LOG.info('--- Building image END (do_build_image) ---')
|
||||
except Exception as exc:
|
||||
LOG.error('Failed to build image: %s', exc)
|
||||
raise
|
||||
finally:
|
||||
LOG.debug('Finally: stopping processes inside chroot: %s', chroot)
|
||||
|
||||
container_size = os.path.getsize(img_containerized)
|
||||
container_md5 = utils.calculate_md5(
|
||||
img_containerized, container_size)
|
||||
metadata.setdefault('images', []).append({
|
||||
'raw_md5': raw_md5,
|
||||
'raw_size': raw_size,
|
||||
'raw_name': None,
|
||||
'container_name': os.path.basename(img_containerized),
|
||||
'container_md5': container_md5,
|
||||
'container_size': container_size,
|
||||
'container': image.container,
|
||||
'format': image.format})
|
||||
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
|
||||
bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
|
||||
LOG.debug('Finally: umounting procfs %s', proc_path)
|
||||
fu.umount_fs(proc_path)
|
||||
LOG.debug('Finally: umounting chroot tree %s', chroot)
|
||||
self.umount_target(chroot, pseudo=False, try_lazy_umount=False)
|
||||
for image in self.driver.image_scheme.images:
|
||||
LOG.debug('Finally: detaching loop device: %s',
|
||||
str(image.target_device))
|
||||
try:
|
||||
bu.deattach_loop(str(image.target_device))
|
||||
except errors.ProcessExecutionError as e:
|
||||
LOG.warning('Error occured while trying to detach '
|
||||
'loop device %s. Error message: %s',
|
||||
str(image.target_device), e)
|
||||
|
||||
# NOTE(kozhukalov): implement abstract publisher
|
||||
LOG.debug('Image metadata: %s', metadata)
|
||||
with open(self.driver.metadata_uri.split('file://', 1)[1], 'w') as f:
|
||||
yaml.safe_dump(metadata, stream=f)
|
||||
LOG.info('--- Building image END (do_build_image) ---')
|
||||
LOG.debug('Finally: removing temporary file: %s',
|
||||
image.img_tmp_file)
|
||||
try:
|
||||
os.unlink(image.img_tmp_file)
|
||||
except OSError:
|
||||
LOG.debug('Finally: file %s seems does not exist '
|
||||
'or can not be removed', image.img_tmp_file)
|
||||
LOG.debug('Finally: removing chroot directory: %s', chroot)
|
||||
try:
|
||||
os.rmdir(chroot)
|
||||
except OSError:
|
||||
LOG.debug('Finally: directory %s seems does not exist '
|
||||
'or can not be removed', chroot)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import testtools
|
||||
|
||||
import mock
|
||||
|
|
@ -176,30 +177,44 @@ class BuildUtilsTestCase(testtools.TestCase):
|
|||
mock_clean.assert_called_once_with('chroot')
|
||||
mock_path.join.assert_called_once_with('chroot', 'etc/shadow')
|
||||
|
||||
@mock.patch('fuel_agent.utils.build_utils.open',
|
||||
create=True, new_callable=mock.mock_open)
|
||||
@mock.patch('fuel_agent.utils.build_utils.time.sleep')
|
||||
@mock.patch.object(os, 'kill')
|
||||
@mock.patch.object(os, 'readlink', return_value='chroot')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_send_signal_to_chrooted_processes(self, mock_exec, mock_link,
|
||||
mock_kill):
|
||||
mock_exec.return_value = ('kernel 951 1641 1700 1920 3210 4104',
|
||||
'')
|
||||
bu.send_signal_to_chrooted_processes('chroot', 'signal')
|
||||
mock_exec.assert_called_once_with('fuser', '-v', 'chroot',
|
||||
check_exit_code=False)
|
||||
def test_stop_chrooted_processes(self, mock_exec, mock_link,
|
||||
mock_kill, mock_sleep, mock_open):
|
||||
mock_exec.side_effect = [
|
||||
('kernel 951 1641 1700 1920 3210 4104', ''),
|
||||
('kernel 951 1641 1700', ''),
|
||||
('', '')]
|
||||
mock_exec_expected_calls = \
|
||||
[mock.call('fuser', '-v', 'chroot', check_exit_code=False)] * 3
|
||||
|
||||
bu.stop_chrooted_processes('chroot', signal=signal.SIGTERM)
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
|
||||
expected_mock_link_calls = [
|
||||
mock.call('/proc/951/root'),
|
||||
mock.call('/proc/1641/root'),
|
||||
mock.call('/proc/1700/root'),
|
||||
mock.call('/proc/1920/root'),
|
||||
mock.call('/proc/3210/root'),
|
||||
mock.call('/proc/4104/root')]
|
||||
mock.call('/proc/4104/root'),
|
||||
mock.call('/proc/951/root'),
|
||||
mock.call('/proc/1641/root'),
|
||||
mock.call('/proc/1700/root')]
|
||||
expected_mock_kill_calls = [
|
||||
mock.call(951, 'signal'),
|
||||
mock.call(1641, 'signal'),
|
||||
mock.call(1700, 'signal'),
|
||||
mock.call(1920, 'signal'),
|
||||
mock.call(3210, 'signal'),
|
||||
mock.call(4104, 'signal')]
|
||||
mock.call(951, signal.SIGTERM),
|
||||
mock.call(1641, signal.SIGTERM),
|
||||
mock.call(1700, signal.SIGTERM),
|
||||
mock.call(1920, signal.SIGTERM),
|
||||
mock.call(3210, signal.SIGTERM),
|
||||
mock.call(4104, signal.SIGTERM),
|
||||
mock.call(951, signal.SIGTERM),
|
||||
mock.call(1641, signal.SIGTERM),
|
||||
mock.call(1700, signal.SIGTERM)]
|
||||
self.assertEqual(expected_mock_link_calls, mock_link.call_args_list)
|
||||
self.assertEqual(expected_mock_kill_calls, mock_kill.call_args_list)
|
||||
|
||||
|
|
@ -256,8 +271,13 @@ class BuildUtilsTestCase(testtools.TestCase):
|
|||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_deattach_loop(self, mock_exec):
|
||||
bu.deattach_loop('loop')
|
||||
mock_exec.assert_called_once_with('losetup', '-d', 'loop')
|
||||
mock_exec.return_value = ('/dev/loop0: [fd03]:130820 (/dev/loop0)', '')
|
||||
bu.deattach_loop('/dev/loop0', check_exit_code='Fake')
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('losetup', '-a'),
|
||||
mock.call('losetup', '-d', '/dev/loop0', check_exit_code='Fake')
|
||||
]
|
||||
self.assertEqual(mock_exec.call_args_list, mock_exec_expected_calls)
|
||||
|
||||
@mock.patch.object(hu, 'parse_simple_kv')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
|
|
|
|||
|
|
@ -92,16 +92,27 @@ class TestFSUtils(test_base.BaseTestCase):
|
|||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_umount_fs_ok(self, mock_exec):
|
||||
fu.umount_fs('/fake')
|
||||
expected_calls = [
|
||||
mock.call('mountpoint', '-q', '/fake', check_exit_code=[0]),
|
||||
mock.call('umount', '/fake', check_exit_code=[0])
|
||||
]
|
||||
self.assertEqual(expected_calls, mock_exec.call_args_list)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_umount_fs_not_mounted(self, mock_exec):
|
||||
mock_exec.side_effect = errors.ProcessExecutionError
|
||||
fu.umount_fs('/fake')
|
||||
mock_exec.assert_called_once_with(
|
||||
'umount', '/fake', check_exit_code=[0])
|
||||
'mountpoint', '-q', '/fake', check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_umount_fs_error(self, mock_exec):
|
||||
mock_exec.side_effect = [
|
||||
errors.ProcessExecutionError('message'), ('', '')]
|
||||
fu.umount_fs('/fake')
|
||||
None, errors.ProcessExecutionError('message'), ('', '')]
|
||||
fu.umount_fs('/fake', try_lazy_umount=True)
|
||||
expected_calls = [
|
||||
mock.call('mountpoint', '-q', '/fake', check_exit_code=[0]),
|
||||
mock.call('umount', '/fake', check_exit_code=[0]),
|
||||
mock.call('umount', '-l', '/fake', check_exit_code=[0])
|
||||
]
|
||||
|
|
|
|||
|
|
@ -337,12 +337,11 @@ class TestManager(test_base.BaseTestCase):
|
|||
@mock.patch('fuel_agent.manager.open',
|
||||
create=True, new_callable=mock.mock_open)
|
||||
@mock.patch('fuel_agent.manager.tempfile.mkdtemp')
|
||||
@mock.patch('fuel_agent.manager.time.sleep')
|
||||
@mock.patch('fuel_agent.manager.yaml.safe_dump')
|
||||
@mock.patch.object(manager.Manager, 'mount_target')
|
||||
@mock.patch.object(manager.Manager, 'umount_target')
|
||||
def test_do_build_image(self, mock_umount_target, mock_mount_target,
|
||||
mock_yaml_dump, mock_sleep, mock_mkdtemp,
|
||||
mock_yaml_dump, mock_mkdtemp,
|
||||
mock_open, mock_shutil_move, mock_os,
|
||||
mock_utils, mock_fu, mock_bu):
|
||||
|
||||
|
|
@ -381,6 +380,8 @@ class TestManager(test_base.BaseTestCase):
|
|||
'fakemd5_raw_boot', 'fakemd5_gzip_boot']
|
||||
mock_utils.calculate_md5.side_effect = md5_side
|
||||
mock_bu.containerize.side_effect = ['/tmp/img.gz', '/tmp/img-boot.gz']
|
||||
mock_bu.stop_chrooted_processes.side_effect = [
|
||||
False, True, False, True]
|
||||
|
||||
self.mgr.do_build_image()
|
||||
self.assertEqual(
|
||||
|
|
@ -450,15 +451,20 @@ class TestManager(test_base.BaseTestCase):
|
|||
mock_bu.run_apt_get.assert_called_once_with(
|
||||
'/tmp/imgdir', packages=['fakepackage1', 'fakepackage2'])
|
||||
mock_bu.do_post_inst.assert_called_once_with('/tmp/imgdir')
|
||||
signal_calls = mock_bu.send_signal_to_chrooted_processes.call_args_list
|
||||
self.assertEqual([mock.call('/tmp/imgdir', signal.SIGTERM),
|
||||
mock.call('/tmp/imgdir', signal.SIGKILL)],
|
||||
signal_calls = mock_bu.stop_chrooted_processes.call_args_list
|
||||
self.assertEqual(2 * [mock.call('/tmp/imgdir', signal=signal.SIGTERM),
|
||||
mock.call('/tmp/imgdir', signal=signal.SIGKILL)],
|
||||
signal_calls)
|
||||
mock_sleep.assert_called_once_with(2)
|
||||
mock_fu.umount_fs.assert_called_once_with('/tmp/imgdir/proc')
|
||||
mock_umount_target.assert_called_once_with('/tmp/imgdir', pseudo=False)
|
||||
self.assertEqual([mock.call('/dev/loop0'), mock.call('/dev/loop1')],
|
||||
mock_bu.deattach_loop.call_args_list)
|
||||
self.assertEqual(
|
||||
[mock.call('/tmp/imgdir/proc')] * 2,
|
||||
mock_fu.umount_fs.call_args_list)
|
||||
self.assertEqual(
|
||||
[mock.call(
|
||||
'/tmp/imgdir', try_lazy_umount=False, pseudo=False)] * 2,
|
||||
mock_umount_target.call_args_list)
|
||||
self.assertEqual(
|
||||
[mock.call('/dev/loop0'), mock.call('/dev/loop1')] * 2,
|
||||
mock_bu.deattach_loop.call_args_list)
|
||||
self.assertEqual([mock.call('/tmp/img'), mock.call('/tmp/img-boot')],
|
||||
mock_bu.shrink_sparse_file.call_args_list)
|
||||
self.assertEqual([mock.call('/tmp/img'),
|
||||
|
|
|
|||
|
|
@ -16,8 +16,10 @@ import gzip
|
|||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal as sig
|
||||
import stat
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import six
|
||||
import yaml
|
||||
|
|
@ -143,13 +145,23 @@ def suppress_services_start(chroot):
|
|||
os.fchmod(f.fileno(), 0o755)
|
||||
|
||||
|
||||
def clean_dirs(chroot, dirs):
|
||||
def clean_dirs(chroot, dirs, delete=False):
|
||||
"""Method is used to clean directories content
|
||||
or remove directories themselfs.
|
||||
|
||||
:param chroot: Root directory where to look for subdirectories
|
||||
:param dirs: List of directories to clean/remove (Relative to chroot)
|
||||
:param delete: (Boolean) If True, directories will be removed
|
||||
(Default: False)
|
||||
"""
|
||||
for d in dirs:
|
||||
path = os.path.join(chroot, d)
|
||||
if os.path.isdir(path):
|
||||
LOG.debug('Removing dir: %s', path)
|
||||
shutil.rmtree(path)
|
||||
os.makedirs(path)
|
||||
LOG.debug('Removed dir: %s', path)
|
||||
if not delete:
|
||||
LOG.debug('Creating empty dir: %s', path)
|
||||
os.makedirs(path)
|
||||
|
||||
|
||||
def remove_files(chroot, files):
|
||||
|
|
@ -192,17 +204,71 @@ def do_post_inst(chroot):
|
|||
clean_apt_settings(chroot)
|
||||
|
||||
|
||||
def send_signal_to_chrooted_processes(chroot, signal):
|
||||
"""Sends signal to all processes, which are running inside of chroot."""
|
||||
for p in utils.execute('fuser', '-v', chroot,
|
||||
check_exit_code=False)[0].split():
|
||||
try:
|
||||
pid = int(p)
|
||||
if os.readlink('/proc/%s/root' % pid) == chroot:
|
||||
LOG.debug('Sending %s to chrooted process %s', signal, pid)
|
||||
os.kill(pid, signal)
|
||||
except (OSError, ValueError):
|
||||
LOG.warning('Skipping non pid %s from fuser output' % p)
|
||||
def stop_chrooted_processes(chroot, signal=sig.SIGTERM,
|
||||
attempts=10, attempts_delay=2):
|
||||
"""Sends signal to all processes, which are running inside chroot.
|
||||
It tries several times until all processes die. If at some point there
|
||||
are no running processes found, it returns True.
|
||||
|
||||
:param chroot: Process root directory.
|
||||
:param signal: Which signal to send to processes. It must be either
|
||||
SIGTERM or SIGKILL. (Default: SIGTERM)
|
||||
:param attempts: Number of attempts (Default: 10)
|
||||
:param attempts_delay: Delay between attempts (Default: 2)
|
||||
"""
|
||||
|
||||
if signal not in (sig.SIGTERM, sig.SIGKILL):
|
||||
raise ValueError('Signal must be either SIGTERM or SIGKILL')
|
||||
|
||||
def get_running_processes():
|
||||
return utils.execute(
|
||||
'fuser', '-v', chroot, check_exit_code=False)[0].split()
|
||||
|
||||
for i in six.moves.range(attempts):
|
||||
running_processes = get_running_processes()
|
||||
if not running_processes:
|
||||
LOG.debug('There are no running processes in %s ', chroot)
|
||||
return True
|
||||
for p in running_processes:
|
||||
try:
|
||||
pid = int(p)
|
||||
if os.readlink('/proc/%s/root' % pid) == chroot:
|
||||
LOG.debug('Sending %s to chrooted process %s', signal, pid)
|
||||
os.kill(pid, signal)
|
||||
except (OSError, ValueError) as e:
|
||||
cmdline = ''
|
||||
pid = p
|
||||
try:
|
||||
with open('/proc/%s/cmdline' % pid) as f:
|
||||
cmdline = f.read()
|
||||
except Exception:
|
||||
LOG.debug('Can not read cmdline for pid=%s', pid)
|
||||
LOG.warning('Exception while sending signal: '
|
||||
'pid: %s cmdline: %s message: %s. Skipping it.',
|
||||
pid, cmdline, e)
|
||||
|
||||
# First of all, signal delivery is asynchronous.
|
||||
# Just because the signal has been sent doesn't
|
||||
# mean the kernel will deliver it instantly
|
||||
# (the target process might be uninterruptible at the moment).
|
||||
# Secondly, exiting might take a while (the process might have
|
||||
# some data to fsync, etc)
|
||||
LOG.debug('Attempt %s. Waiting for %s seconds', i + 1, attempts_delay)
|
||||
time.sleep(attempts_delay)
|
||||
|
||||
running_processes = get_running_processes()
|
||||
if running_processes:
|
||||
for pid in running_processes:
|
||||
cmdline = ''
|
||||
try:
|
||||
with open('/proc/%s/cmdline' % pid) as f:
|
||||
cmdline = f.read()
|
||||
except Exception:
|
||||
LOG.debug('Can not read cmdline for pid=%s', pid)
|
||||
LOG.warning('Process is still running: pid=%s cmdline: %s',
|
||||
pid, cmdline)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def get_free_loop_device(
|
||||
|
|
@ -243,8 +309,17 @@ def attach_file_to_loop(filename, loop):
|
|||
utils.execute('losetup', loop, filename)
|
||||
|
||||
|
||||
def deattach_loop(loop):
|
||||
utils.execute('losetup', '-d', loop)
|
||||
def deattach_loop(loop, check_exit_code=[0]):
|
||||
LOG.debug('Trying to figure out if loop device %s is attached', loop)
|
||||
output = utils.execute('losetup', '-a')[0]
|
||||
for line in output.split('\n'):
|
||||
# output lines are assumed to have the following format
|
||||
# /dev/loop0: [fd03]:130820 (/dev/loop0)
|
||||
if loop == line.split(':')[0]:
|
||||
LOG.debug('Loop device %s seems to be attached. '
|
||||
'Trying to detach.', loop)
|
||||
utils.execute('losetup', '-d', loop,
|
||||
check_exit_code=check_exit_code)
|
||||
|
||||
|
||||
def shrink_sparse_file(filename):
|
||||
|
|
|
|||
|
|
@ -58,12 +58,20 @@ def mount_bind(chroot, path, path2=None):
|
|||
check_exit_code=[0])
|
||||
|
||||
|
||||
def umount_fs(fs_mount):
|
||||
def umount_fs(fs_mount, try_lazy_umount=False):
|
||||
try:
|
||||
utils.execute('mountpoint', '-q', fs_mount, check_exit_code=[0])
|
||||
except errors.ProcessExecutionError:
|
||||
LOG.warning('%s is not a mountpoint, skipping umount', fs_mount)
|
||||
else:
|
||||
LOG.debug('Trying to umount {0}'.format(fs_mount))
|
||||
utils.execute('umount', fs_mount, check_exit_code=[0])
|
||||
except errors.ProcessExecutionError as e:
|
||||
LOG.warning('Error while umounting {0} '
|
||||
'exc={1}'.format(fs_mount, e.message))
|
||||
LOG.debug('Trying lazy umounting {0}'.format(fs_mount))
|
||||
utils.execute('umount', '-l', fs_mount, check_exit_code=[0])
|
||||
try:
|
||||
utils.execute('umount', fs_mount, check_exit_code=[0])
|
||||
except errors.ProcessExecutionError as e:
|
||||
if try_lazy_umount:
|
||||
LOG.warning('Error while umounting {0} '
|
||||
'exc={1}'.format(fs_mount, e.message))
|
||||
LOG.debug('Trying lazy umounting {0}'.format(fs_mount))
|
||||
utils.execute('umount', '-l', fs_mount, check_exit_code=[0])
|
||||
else:
|
||||
raise
|
||||
|
|
|
|||
Loading…
Reference in New Issue