openstack
/
fuel-qa
Code Issues Proposed changes

Reboot the master after an update 

Patch adds option for the reboot
the master node after updates are installed

Change-Id: If347f57b70cb8cc959dd2340f7ae071a556a0733
This commit is contained in:
Alexander Kurenyshev 2016-12-12 19:09:08 +03:00
parent e9a4009a77
commit a043cc95dc
5 changed files with 89 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
fuelweb_test/helpers/fuel_actions.py
View File

@ -19,6 +19,7 @@ from devops.helpers.helpers import wait
from devops.models import DiskDevice
from devops.models import Node
from devops.models import Volume
from proboscis.asserts import assert_equal
from proboscis.asserts import assert_true
from core.helpers.log_helpers import logwrap
@ -45,6 +46,7 @@ class BaseActions(object):
def __init__(self):
self.ssh_manager = SSHManager()
self.admin_ip = self.ssh_manager.admin_ip
self.__new_packages = None
def __repr__(self):
klass, obj_id = type(self), hex(id(self))
@ -59,6 +61,14 @@ class BaseActions(object):
err_msg="Failed to restart service {!r}, please inspect logs for "
"details".format(service))
@property
def new_packages(self):
return self.__new_packages
@new_packages.setter
def new_packages(self, packages):
self.__new_packages = packages
class AdminActions(BaseActions):
""" All actions relating to the admin node."""
@ -304,7 +314,10 @@ class AdminActions(BaseActions):
self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip,
command=install_command)
logger.info('Requesting packages to be updated')
self.new_packages = self.get_packages_list()
logger.debug('Got the list of packages to be updated: {}'.format(
self.new_packages))
logger.info('prepare Fuel node for updating')
prepare_command = 'update-prepare prepare master'
@ -319,10 +332,45 @@ class AdminActions(BaseActions):
ip=self.ssh_manager.admin_ip,
command=update_command)
logger.info('Requesting installed packages')
installed_packages = self.get_packages_list('installed')
logger.debug('Got the list of installed packages: {}'.format(
installed_packages))
self.check_packages(installed_packages)
logger.info('Update successful')
def get_packages_list(self, packages_list='updates'):
cmd = 'yum list {}'.format(packages_list)
admin_ip = self.ssh_manager.admin_ip
res = self.ssh_manager.check_call(admin_ip, cmd).stdout
tmp = '{} Packages\n'.format(
'Updated' if packages_list == 'updates' else 'Installed')
index = res.index(tmp)
res = res[index + 1::]
# the 'magic' below allows us to get the full list of packages
# we should use it because the command returns too long strings
raw = ''.join(res).replace('\n', ' ').split()
i = 0
packages = {}
while i < len(raw):
packages[raw[i]] = raw[i + 1]
i += 3
return packages
def check_packages(self, packages):
notupdated_packages = []
for package in self.new_packages:
if packages[package] != self.new_packages[package]:
notupdated_packages.append(package)
msg = "These packages weren't updated {}".format(notupdated_packages)
assert_equal(notupdated_packages, [], msg)
def prepare_admin_node_for_mos_mu(self):
"""Prepare master node for update via mos-playbooks tool"""
logger.info('Requesting packages to be updated')
self.new_packages = self.get_packages_list()
logger.debug('Got the list of packages to be updated: {}'.format(
self.new_packages))
mos_mu_path = 'cd {} &&'.format(MOS_MU_PATH)
logger.info('Searching for mos-playbooks and mos-release packages')
@ -383,6 +431,11 @@ class AdminActions(BaseActions):
ip=self.ssh_manager.admin_ip,
command=command)
logger.info('Requesting installed packages')
installed_packages = self.get_packages_list('installed')
logger.debug('Got the list of installed packages: {}'.format(
installed_packages))
self.check_packages(installed_packages)
logger.info('Update Fuel node was successful')

6
fuelweb_test/models/environment.py
View File

@ -430,7 +430,7 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
if settings.SHOW_FUELMENU:
self.wait_for_fuelmenu()
else:
self.wait_for_provisioning()
self.wait_ssh_available()
try:
provision_admin(admin)
@ -494,8 +494,8 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
# pylint: enable=no-self-use
@logwrap
def wait_for_provisioning(self,
timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT):
def wait_ssh_available(self,
timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT):
# TODO(astudenov): add timeout_msg
wait_pass(lambda: tcp_ping_(
self.d_env.nodes(

2
fuelweb_test/settings.py
View File

@ -673,6 +673,8 @@ UPDATE_FUEL_MIRROR = os.environ.get("UPDATE_FUEL_MIRROR", '').split()
UPDATE_MASTER = get_var_as_bool('UPDATE_MASTER', False)
UPDATE_MASTER_VIA_MOS_MU = get_var_as_bool('UPDATE_MASTER_VIA_MOS_MU', True)
REBOOT_MASTER_AFTER_UPDATE = get_var_as_bool(
'REBOOT_MASTER_AFTER_UPDATE', True)
EMC_PLUGIN_PATH = os.environ.get('EMC_PLUGIN_PATH')
EMC_SP_A_IP = os.environ.get('EMC_SP_A_IP')

29
fuelweb_test/tests/base_test_case.py
View File

@ -19,12 +19,14 @@ from proboscis import TestProgram
from proboscis import SkipTest
from proboscis import test
from proboscis.asserts import assert_equal
from proboscis.asserts import assert_true
from fuelweb_test import logger
from fuelweb_test import settings
from fuelweb_test.helpers import replace_repos
from fuelweb_test.helpers.decorators import log_snapshot_after_test
from fuelweb_test.helpers.utils import erase_data_from_hdd
from fuelweb_test.helpers.utils import get_process_uptime
from fuelweb_test.helpers.utils import get_test_method_name
from fuelweb_test.helpers.utils import TimeStat
from fuelweb_test.helpers.utils import YamlEditor
@ -234,6 +236,9 @@ class TestBasic(object):
self.env.admin_actions.admin_install_updates_mos_mu()
else:
self.env.admin_actions.admin_install_updates()
self.admin_reboot_and_wait(settings.REBOOT_MASTER_AFTER_UPDATE)
if settings.MULTIPLE_NETWORKS:
self.env.describe_other_admin_interfaces(
self.env.d_env.nodes().admin)
@ -252,6 +257,30 @@ class TestBasic(object):
if force_ssl:
self.env.enable_force_https(self.ssh_manager.admin_ip)
def admin_reboot_and_wait(self, reboot=False):
if not reboot:
logger.info(
"Reboot is switched off due to "
"REBOOT_MASTER_AFTER_UPDATE={}".format(reboot))
return
admin_node = self.env.d_env.nodes().admin
admin_remote = self.ssh_manager.get_remote(self.ssh_manager.admin_ip)
uptime_old = get_process_uptime(admin_remote, 'systemd')
try:
admin_remote.execute('reboot')
time.sleep(300)
self.env.wait_ssh_available(timeout=300)
except Exception:
logger.info("Reboot failed. Hard resetting admin node",
exc_info=True)
admin_node.reset()
time.sleep(300)
self.env.wait_ssh_available()
# sh_manager.update_connection fails with auth errors, use re-defining
admin_remote = self.ssh_manager.get_remote(self.ssh_manager.admin_ip)
assert_true(uptime_old > get_process_uptime(admin_remote, 'systemd'),
"Uptime was not changed, restart failed")
def reinstall_master_node(self):
"""Erase boot sector and run setup_environment"""
admin = self.env.d_env.get_node(name="admin")

2
system_test/actions/base.py
View File

@ -176,7 +176,7 @@ class PrepareActions(object):
wait(lambda: admin.driver.node_active(admin), 60,
timeout_msg='Centos node failed to start')
logger.info("Waiting for Centos node ssh ready")
self.env.wait_for_provisioning()
self.env.wait_ssh_available()
self.centos_setup_fuel(hostname)