Merge "Always take into account config file values"

This commit is contained in:
Jenkins 2015-07-31 14:13:22 +00:00 committed by Gerrit Code Review
commit 1fe47720ba
6 changed files with 128 additions and 111 deletions

View File

@ -72,6 +72,51 @@ opts = [
default='.fuel-agent-image',
help='Suffix which is used while creating temporary files',
),
cfg.IntOpt(
'grub_timeout',
default=5,
help='Timeout in secs for GRUB'
),
cfg.IntOpt(
'max_loop_devices_count',
default=255,
# NOTE(agordeev): up to 256 loop devices could be allocated up to
# kernel version 2.6.23, and the limit (from version 2.6.24 onwards)
# isn't theoretically present anymore.
help='Maximum allowed loop devices count to use'
),
cfg.IntOpt(
'sparse_file_size',
# XXX: Apparently Fuel configures the node root filesystem to span
# the whole hard drive. However 2 GB filesystem created with default
# options can grow at most to 2 TB (1024x its initial size). This
# maximal size can be configured by mke2fs -E resize=NNN option,
# however the version of e2fsprogs shipped with CentOS 6.[65] seems
# to silently ignore the `resize' option. Therefore make the initial
# filesystem a bit bigger so it can grow to 8 TB.
default=8192,
help='Size of sparse file in MiBs'
),
cfg.IntOpt(
'loop_device_major_number',
default=7,
help='System-wide major number for loop device'
),
cfg.IntOpt(
'fetch_packages_attempts',
default=10,
help='Maximum allowed debootstrap/apt-get attempts to execute'
),
cfg.StrOpt(
'allow_unsigned_file',
default='allow_unsigned_packages',
help='File where to store apt setting for unsigned packages'
),
cfg.StrOpt(
'force_ipv4_file',
default='force_ipv4',
help='File where to store apt setting for forcing IPv4 usage'
),
]
cli_opts = [
@ -381,12 +426,14 @@ class Manager(object):
if grub.version == 1:
gu.grub1_cfg(kernel=kernel, initrd=initrd,
kernel_params=grub.kernel_params, chroot=chroot)
kernel_params=grub.kernel_params, chroot=chroot,
grub_timeout=CONF.grub_timeout)
gu.grub1_install(install_devices, boot_device, chroot=chroot)
else:
# TODO(kozhukalov): implement which kernel to use by default
# Currently only grub1_cfg accepts kernel and initrd parameters.
gu.grub2_cfg(kernel_params=grub.kernel_params, chroot=chroot)
gu.grub2_cfg(kernel_params=grub.kernel_params, chroot=chroot,
grub_timeout=CONF.grub_timeout)
gu.grub2_install(install_devices, chroot=chroot)
# FIXME(agordeev) There's no convenient way to perfrom NIC remapping in
@ -502,7 +549,8 @@ class Manager(object):
LOG.debug('Creating temporary sparsed file for the '
'image: %s', image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix,
size=CONF.sparse_file_size)
LOG.debug('Temporary file: %s', img_tmp_file)
# we need to remember those files
@ -510,7 +558,9 @@ class Manager(object):
image.img_tmp_file = img_tmp_file
LOG.debug('Looking for a free loop device')
image.target_device.name = bu.get_free_loop_device()
image.target_device.name = bu.get_free_loop_device(
loop_device_major_number=CONF.loop_device_major_number,
max_loop_devices_count=CONF.max_loop_devices_count)
LOG.debug('Attaching temporary image file to free loop device')
bu.attach_file_to_loop(img_tmp_file, str(image.target_device))
@ -547,14 +597,17 @@ class Manager(object):
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
LOG.debug('Installing base operating system using debootstrap')
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot)
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot,
attempts=CONF.fetch_packages_attempts)
# APT-GET
LOG.debug('Configuring apt inside chroot')
LOG.debug('Setting environment variables')
bu.set_apt_get_env()
LOG.debug('Allowing unauthenticated repos')
bu.pre_apt_get(chroot)
bu.pre_apt_get(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
for repo in self.driver.operating_system.repos:
LOG.debug('Adding repository source: name={name}, uri={uri},'
@ -600,10 +653,13 @@ class Manager(object):
LOG.debug('Installing packages using apt-get: %s',
' '.join(packages))
bu.run_apt_get(chroot, packages=packages)
bu.run_apt_get(chroot, packages=packages,
attempts=CONF.fetch_packages_attempts)
LOG.debug('Post-install OS configuration')
bu.do_post_inst(chroot)
bu.do_post_inst(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
LOG.debug('Making sure there are no running processes '
'inside chroot before trying to umount chroot')
@ -644,7 +700,8 @@ class Manager(object):
LOG.debug('Containerizing temporary image file: %s',
image.img_tmp_file)
img_tmp_containerized = bu.containerize(
image.img_tmp_file, image.container)
image.img_tmp_file, image.container,
chunk_size=CONF.data_chunk_size)
img_containerized = image.uri.split('file://', 1)[1]
# NOTE(kozhukalov): implement abstract publisher

View File

@ -17,7 +17,6 @@ import shutil
import signal
import mock
from oslo.config import cfg
import unittest2
from fuel_agent import errors
@ -26,9 +25,6 @@ from fuel_agent.utils import hardware as hu
from fuel_agent.utils import utils
CONF = cfg.CONF
class BuildUtilsTestCase(unittest2.TestCase):
_fake_ubuntu_release = '''
@ -169,7 +165,8 @@ class BuildUtilsTestCase(unittest2.TestCase):
def test_do_post_inst(self, mock_exec, mock_files, mock_clean, mock_path,
mock_open):
mock_path.join.return_value = 'fake_path'
bu.do_post_inst('chroot')
bu.do_post_inst('chroot', allow_unsigned_file='fake_unsigned',
force_ipv4_file='fake_force_ipv4')
file_handle_mock = mock_open.return_value.__enter__.return_value
file_handle_mock.write.assert_called_once_with('manual\n')
mock_exec_expected_calls = [
@ -179,7 +176,9 @@ class BuildUtilsTestCase(unittest2.TestCase):
mock.call('chroot', 'chroot', 'update-rc.d', 'puppet', 'disable')]
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
mock_files.assert_called_once_with('chroot', ['usr/sbin/policy-rc.d'])
mock_clean.assert_called_once_with('chroot')
mock_clean.assert_called_once_with('chroot',
allow_unsigned_file='fake_unsigned',
force_ipv4_file='fake_force_ipv4')
mock_path_join_expected_calls = [
mock.call('chroot', 'etc/shadow'),
mock.call('chroot', 'etc/init/mcollective.override')]
@ -455,18 +454,21 @@ class BuildUtilsTestCase(unittest2.TestCase):
def test_pre_apt_get(self, mock_path, mock_clean):
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
file_handle_mock = mock_open.return_value.__enter__.return_value
bu.pre_apt_get('chroot')
bu.pre_apt_get('chroot', allow_unsigned_file='fake_unsigned',
force_ipv4_file='fake_force_ipv4')
expected_calls = [
mock.call('APT::Get::AllowUnauthenticated 1;\n'),
mock.call('Acquire::ForceIPv4 "true";\n')]
self.assertEqual(expected_calls,
file_handle_mock.write.call_args_list)
mock_clean.assert_called_once_with('chroot')
mock_clean.assert_called_once_with('chroot',
allow_unsigned_file='fake_unsigned',
force_ipv4_file='fake_force_ipv4')
expected_join_calls = [
mock.call('chroot', 'etc/apt/apt.conf.d',
CONF.allow_unsigned_file),
'fake_unsigned'),
mock.call('chroot', 'etc/apt/apt.conf.d',
CONF.force_ipv4_file)]
'fake_force_ipv4')]
self.assertEqual(expected_join_calls, mock_path.join.call_args_list)
@mock.patch('gzip.open')

View File

@ -71,7 +71,7 @@ class TestManager(unittest2.TestCase):
mock_gu.grub1_cfg.assert_called_once_with(
kernel_params='fake_kernel_params root=UUID= ',
initrd='guessed_initrd', kernel='guessed_kernel',
chroot='/tmp/target')
chroot='/tmp/target', grub_timeout=5)
mock_gu.grub1_install.assert_called_once_with(
['/dev/sda', '/dev/sdb', '/dev/sdc'],
'/dev/sda3', chroot='/tmp/target')
@ -100,7 +100,8 @@ class TestManager(unittest2.TestCase):
self.assertFalse(mock_gu.grub2_install.called)
mock_gu.grub1_cfg.assert_called_once_with(
kernel_params='fake_kernel_params root=UUID= ',
initrd='initrd_name', kernel='kernel_name', chroot='/tmp/target')
initrd='initrd_name', kernel='kernel_name', chroot='/tmp/target',
grub_timeout=5)
mock_gu.grub1_install.assert_called_once_with(
['/dev/sda', '/dev/sdb', '/dev/sdc'],
'/dev/sda3', chroot='/tmp/target')
@ -183,7 +184,8 @@ class TestManager(unittest2.TestCase):
'nomodeset root=UUID=fake_UUID ',
initrd='guessed_initrd',
chroot='/tmp/target',
kernel='guessed_kernel')
kernel='guessed_kernel',
grub_timeout=5)
mock_gu.grub1_install.assert_called_once_with(
['/dev/sda', '/dev/sdb', '/dev/sdc'],
'/dev/sda3', chroot='/tmp/target')
@ -207,7 +209,7 @@ class TestManager(unittest2.TestCase):
mock_gu.grub2_cfg.assert_called_once_with(
kernel_params=' console=ttyS0,9600 console=tty0 rootdelay=90 '
'nomodeset root=UUID=fake_UUID ',
chroot='/tmp/target')
chroot='/tmp/target', grub_timeout=5)
mock_gu.grub2_install.assert_called_once_with(
['/dev/sda', '/dev/sdb', '/dev/sdc'],
chroot='/tmp/target')
@ -798,10 +800,14 @@ class TestImageBuild(unittest2.TestCase):
mock.call('/fake/img-boot.img.gz')],
mock_os.path.exists.call_args_list)
self.assertEqual([mock.call(dir=CONF.image_build_dir,
suffix=CONF.image_build_suffix)] * 2,
suffix=CONF.image_build_suffix,
size=CONF.sparse_file_size)] * 2,
mock_bu.create_sparse_tmp_file.call_args_list)
self.assertEqual([mock.call()] * 2,
mock_bu.get_free_loop_device.call_args_list)
self.assertEqual(
[mock.call(loop_device_major_number=CONF.loop_device_major_number,
max_loop_devices_count=CONF.max_loop_devices_count),
] * 2,
mock_bu.get_free_loop_device.call_args_list)
self.assertEqual([mock.call('/tmp/img', '/dev/loop0'),
mock.call('/tmp/img-boot', '/dev/loop1')],
mock_bu.attach_file_to_loop.call_args_list)
@ -817,9 +823,12 @@ class TestImageBuild(unittest2.TestCase):
self.assertEqual([mock.call('/tmp/imgdir')] * 2,
mock_bu.suppress_services_start.call_args_list)
mock_bu.run_debootstrap.assert_called_once_with(
uri='http://fakeubuntu', suite='trusty', chroot='/tmp/imgdir')
uri='http://fakeubuntu', suite='trusty', chroot='/tmp/imgdir',
attempts=CONF.fetch_packages_attempts)
mock_bu.set_apt_get_env.assert_called_once_with()
mock_bu.pre_apt_get.assert_called_once_with('/tmp/imgdir')
mock_bu.pre_apt_get.assert_called_once_with(
'/tmp/imgdir', allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
self.assertEqual([
mock.call(name='ubuntu',
uri='http://fakeubuntu',
@ -864,8 +873,12 @@ class TestImageBuild(unittest2.TestCase):
mock_utils.execute.call_args_list)
mock_fu.mount_bind.assert_called_once_with('/tmp/imgdir', '/proc')
mock_bu.run_apt_get.assert_called_once_with(
'/tmp/imgdir', packages=['fakepackage1', 'fakepackage2'])
mock_bu.do_post_inst.assert_called_once_with('/tmp/imgdir')
'/tmp/imgdir', packages=['fakepackage1', 'fakepackage2'],
attempts=CONF.fetch_packages_attempts)
mock_bu.do_post_inst.assert_called_once_with(
'/tmp/imgdir', allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
signal_calls = mock_bu.stop_chrooted_processes.call_args_list
self.assertEqual(2 * [mock.call('/tmp/imgdir', signal=signal.SIGTERM),
mock.call('/tmp/imgdir', signal=signal.SIGKILL)],
@ -892,8 +905,10 @@ class TestImageBuild(unittest2.TestCase):
mock.call('/tmp/img-boot', 10),
mock.call('/fake/img-boot.img.gz', 1)],
mock_utils.calculate_md5.call_args_list)
self.assertEqual([mock.call('/tmp/img', 'gzip'),
mock.call('/tmp/img-boot', 'gzip')],
self.assertEqual([mock.call('/tmp/img', 'gzip',
chunk_size=CONF.data_chunk_size),
mock.call('/tmp/img-boot', 'gzip',
chunk_size=CONF.data_chunk_size)],
mock_bu.containerize.call_args_list)
mock_open.assert_called_once_with('/fake/img.yaml', 'w')
self.assertEqual(

View File

@ -173,8 +173,10 @@ class ForwardFileStream(Target):
self.chunk = None
self.position = position
def read(self, length=CONF.data_chunk_size):
# NOTE(kozhukalov): default lenght = 1048576 is not usual behaviour,
def read(self, length=None):
if length is None:
length = CONF.data_chunk_size
# NOTE(kozhukalov): default length = 1048576 is not usual behaviour,
# but that is ok for our use case.
if self.closed:
raise ValueError('I/O operation on closed file')

View File

@ -24,8 +24,6 @@ import time
import six
import yaml
from oslo.config import cfg
from fuel_agent import errors
from fuel_agent.openstack.common import log as logging
from fuel_agent.utils import hardware as hu
@ -33,51 +31,6 @@ from fuel_agent.utils import utils
LOG = logging.getLogger(__name__)
bu_opts = [
cfg.IntOpt(
'max_loop_devices_count',
default=255,
# NOTE(agordeev): up to 256 loop devices could be allocated up to
# kernel version 2.6.23, and the limit (from version 2.6.24 onwards)
# isn't theoretically present anymore.
help='Maximum allowed loop devices count to use'
),
cfg.IntOpt(
'sparse_file_size',
# XXX: Apparently Fuel configures the node root filesystem to span
# the whole hard drive. However 2 GB filesystem created with default
# options can grow at most to 2 TB (1024x its initial size). This
# maximal size can be configured by mke2fs -E resize=NNN option,
# however the version of e2fsprogs shipped with CentOS 6.[65] seems
# to silently ignore the `resize' option. Therefore make the initial
# filesystem a bit bigger so it can grow to 8 TB.
default=8192,
help='Size of sparse file in MiBs'
),
cfg.IntOpt(
'loop_device_major_number',
default=7,
help='System-wide major number for loop device'
),
cfg.IntOpt(
'fetch_packages_attempts',
default=10,
help='Maximum allowed debootstrap/apt-get attempts to execute'
),
cfg.StrOpt(
'allow_unsigned_file',
default='allow_unsigned_packages',
help='File where to store apt setting for unsigned packages'
),
cfg.StrOpt(
'force_ipv4_file',
default='force_ipv4',
help='File where to store apt setting for forcing IPv4 usage'
),
]
CONF = cfg.CONF
CONF.register_opts(bu_opts)
DEFAULT_APT_PATH = {
'sources_file': 'etc/apt/sources.list',
'sources_dir': 'etc/apt/sources.list.d',
@ -91,7 +44,7 @@ ROOT_PASSWORD = '$6$IInX3Cqo$5xytL1VZbZTusOewFnG6couuF0Ia61yS3rbC6P5YbZP2TYcl'\
def run_debootstrap(uri, suite, chroot, arch='amd64', eatmydata=False,
attempts=CONF.fetch_packages_attempts):
attempts=10):
"""Builds initial base system.
debootstrap builds initial base system which is capable to run apt-get.
@ -114,8 +67,7 @@ def set_apt_get_env():
os.environ['LC_ALL'] = os.environ['LANG'] = os.environ['LANGUAGE'] = 'C'
def run_apt_get(chroot, packages, eatmydata=False,
attempts=CONF.fetch_packages_attempts):
def run_apt_get(chroot, packages, eatmydata=False, attempts=10):
"""Runs apt-get install <packages>.
Unlike debootstrap, apt-get has a perfect package dependecies resolver
@ -178,8 +130,8 @@ def remove_files(chroot, files):
LOG.debug('Removed file: %s', path)
def clean_apt_settings(chroot, allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file):
def clean_apt_settings(chroot, allow_unsigned_file='allow_unsigned_packages',
force_ipv4_file='force_ipv4'):
"""Cleans apt settings such as package sources and repo pinning."""
files = [DEFAULT_APT_PATH['sources_file'],
DEFAULT_APT_PATH['preferences_file'],
@ -191,7 +143,8 @@ def clean_apt_settings(chroot, allow_unsigned_file=CONF.allow_unsigned_file,
clean_dirs(chroot, dirs)
def do_post_inst(chroot):
def do_post_inst(chroot, allow_unsigned_file='allow_unsigned_packages',
force_ipv4_file='force_ipv4'):
# NOTE(agordeev): set up password for root
utils.execute('sed', '-i',
's%root:[\*,\!]%root:' + ROOT_PASSWORD + '%',
@ -211,7 +164,8 @@ def do_post_inst(chroot):
# NOTE(agordeev): remove custom policy-rc.d which is needed to disable
# execution of post/pre-install package hooks and start of services
remove_files(chroot, ['usr/sbin/policy-rc.d'])
clean_apt_settings(chroot)
clean_apt_settings(chroot, allow_unsigned_file=allow_unsigned_file,
force_ipv4_file=force_ipv4_file)
def stop_chrooted_processes(chroot, signal=sig.SIGTERM,
@ -282,9 +236,8 @@ def stop_chrooted_processes(chroot, signal=sig.SIGTERM,
return True
def get_free_loop_device(
loop_device_major_number=CONF.loop_device_major_number,
max_loop_devices_count=CONF.max_loop_devices_count):
def get_free_loop_device(loop_device_major_number=7,
max_loop_devices_count=255):
"""Returns the name of free loop device.
It should return the name of free loop device or raise an exception.
@ -305,7 +258,7 @@ def get_free_loop_device(
raise errors.NoFreeLoopDevices('Free loop device not found')
def create_sparse_tmp_file(dir, suffix, size=CONF.sparse_file_size):
def create_sparse_tmp_file(dir, suffix, size=8192):
"""Creates sparse file.
Creates file which consumes disk space more efficiently when the file
@ -472,10 +425,11 @@ def add_apt_preference(name, priority, suite, section, chroot, uri):
f.write('Pin-Priority: {priority}\n'.format(priority=priority))
def pre_apt_get(chroot, allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file):
def pre_apt_get(chroot, allow_unsigned_file='allow_unsigned_packages',
force_ipv4_file='force_ipv4'):
"""It must be called prior run_apt_get."""
clean_apt_settings(chroot)
clean_apt_settings(chroot, allow_unsigned_file=allow_unsigned_file,
force_ipv4_file=force_ipv4_file)
# NOTE(agordeev): allow to install packages without gpg digest
with open(os.path.join(chroot, DEFAULT_APT_PATH['conf_dir'],
allow_unsigned_file), 'w') as f:
@ -485,7 +439,7 @@ def pre_apt_get(chroot, allow_unsigned_file=CONF.allow_unsigned_file,
f.write('Acquire::ForceIPv4 "true";\n')
def containerize(filename, container, chunk_size=CONF.data_chunk_size):
def containerize(filename, container, chunk_size=1048576):
if container == 'gzip':
output_file = filename + '.gz'
with open(filename, 'rb') as f:

View File

@ -16,25 +16,12 @@ import os
import re
import shutil
from oslo.config import cfg
from fuel_agent import errors
from fuel_agent.openstack.common import log as logging
from fuel_agent.utils import utils
LOG = logging.getLogger(__name__)
gu_opts = [
cfg.IntOpt(
'grub_timeout',
default=5,
help='Timeout in secs for GRUB'
),
]
CONF = cfg.CONF
CONF.register_opts(gu_opts)
def guess_grub2_conf(chroot=''):
for filename in ('/boot/grub/grub.cfg', '/boot/grub2/grub.cfg'):
@ -208,7 +195,7 @@ def grub1_stage1(chroot=''):
def grub1_cfg(kernel=None, initrd=None,
kernel_params='', chroot='', grub_timeout=CONF.grub_timeout):
kernel_params='', chroot='', grub_timeout=5):
if not kernel:
kernel = guess_kernel(chroot=chroot)
@ -237,7 +224,7 @@ def grub2_install(install_devices, chroot=''):
utils.execute(*cmd, run_as_root=True, check_exit_code=[0])
def grub2_cfg(kernel_params='', chroot='', grub_timeout=CONF.grub_timeout):
def grub2_cfg(kernel_params='', chroot='', grub_timeout=5):
grub_defaults = chroot + guess_grub2_default(chroot=chroot)
rekerparams = re.compile(r'^.*GRUB_CMDLINE_LINUX=.*')
retimeout = re.compile(r'^.*GRUB_HIDDEN_TIMEOUT=.*')