openstack
/
fuel-qa
Code Issues Proposed changes

Merge "Test compute rebooting after migration of fuel master"

This commit is contained in:
Jenkins 2016-08-04 13:08:22 +00:00 committed by Gerrit Code Review
parent f73341b7c5 8819e26868
commit 2e2d01b7c0
3 changed files with 263 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

220
fuel_tests/tests/test_fuel_migration.py Normal file
View File

@ -0,0 +1,220 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import pytest
from devops.helpers.helpers import get_admin_remote
from devops.helpers.helpers import icmp_ping
from devops.helpers.helpers import _wait
from devops.helpers.helpers import wait
from fuelweb_test import logger
from fuelweb_test import settings
# pylint: disable=no-member
@pytest.fixture(scope='session')
def fuel_master_migration(request):
"""Fixture which migrate Fuel Master to a compute"""
instance = request.node.items[-1].instance
cluster_id = instance._storage['cluster_id']
instance.start_fuel_migration()
instance.check_migration_status()
instance.run_checkers()
instance.manager.fuel_web.verify_network(cluster_id)
instance.manager.fuel_web.run_ostf(cluster_id=cluster_id)
@pytest.mark.get_logs
@pytest.mark.fail_snapshot
@pytest.mark.thread_1
class TestFuelMasterMigrate(object):
compute = None
cluster_config = {
'name': "FuelMasterMigrate",
'mode': settings.DEPLOYMENT_MODE,
'nodes': {
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute'],
'slave-05': ['compute'],
}
}
@pytest.mark.need_ready_cluster
@pytest.mark.usefixtures("fuel_master_migration")
@pytest.mark.test_compute_hard_restart
def test_compute_hard_restart(self):
"""Check Fuel Master node functionality after hard restart of the
compute where Fuel Master node is located
Scenario:
1. Deploy cluster with two computes and three controllers
2. Migrate Fuel Master
3. Hard restart for compute node where Fuel Master node was
migrated to
4. Reconnect to Fuel Master
5. Check status for master's services
6. Run OSTF
"""
cluster_id = self._storage['cluster_id']
fuel_web = self.manager.fuel_web
self.manager.show_step(1)
self.manager.show_step(2)
self.manager.show_step(3)
self.compute_hard_restart()
self.node_rebooted(self.env.get_admin_node_ip())
self.manager.show_step(4)
self.run_checkers()
self.manager.show_step(5)
fuel_web.verify_network(cluster_id)
self.manager.show_step(6)
fuel_web.run_ostf(cluster_id=cluster_id)
@pytest.mark.need_ready_cluster
@pytest.mark.usefixtures("fuel_master_migration")
@pytest.mark.test_compute_warm_restart
def test_compute_warm_restart(self):
"""Check Fuel Master node functionality after warm restart of the
compute where Fuel Master node is located
Scenario:
1. Deploy cluster with two computes and three controllers
2. Migrate Fuel Master
3. Warm restart for compute node where Fuel Master node was
migrated to
4. Reconnect to Fuel Master
5. Check status for master's services
6. Run OSTF
"""
cluster_id = self._storage['cluster_id']
fuel_web = self.manager.fuel_web
self.manager.show_step(1)
self.manager.show_step(2)
self.manager.show_step(3)
self.compute_warm_restart()
self.node_rebooted(self.env.get_admin_node_ip())
self.manager.show_step(4)
self.run_checkers()
self.manager.show_step(5)
fuel_web.verify_network(cluster_id)
self.manager.show_step(6)
fuel_web.run_ostf(cluster_id=cluster_id)
def start_fuel_migration(self):
"""Migrate Fuel Master to a compute"""
# Get a compute to migrate Fuel Master to
cluster_id = self._storage['cluster_id']
fuel_web = self.manager.fuel_web
self.compute = fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['compute'])[0]
logger.info(
'Fuel Master will be migrated to {0} '
'compute'.format(self.compute['name']))
# Start migrating Fuel Master
with self.env.d_env.get_admin_remote() as remote:
remote.execute('fuel-migrate {0} >/dev/null &'.
format(self.compute['ip']))
def check_migration_status(self):
"""Check periodically the status of Fuel Master migration process"""
logger.info(
'Rebooting to begin the data sync process for fuel migrate')
self.node_rebooted(self.env.get_admin_node_ip())
logger.info('Fuel Master is migrating..')
self.node_rebooted(self.env.get_admin_node_ip(), interval=0.5,
timeout=60 * 45)
logger.info('Waiting for appearance of /tmp/migration-done file..')
with get_admin_remote(self.env.d_env) as remote:
wait(lambda: remote.exists("/tmp/migration-done"),
timeout=60 * 5,
timeout_msg="File /tmp/migration-done wasn't appeared")
@staticmethod
def node_rebooted(ip, interval=5, timeout=60 * 15):
wait(lambda: not icmp_ping(ip), interval=interval, timeout=timeout,
timeout_msg=("Node with ip: {} has not become offline after "
"starting reboot").format(ip))
wait(lambda: icmp_ping(ip), interval=interval, timeout=timeout,
timeout_msg="Node with ip: {} has not become online "
"after reboot".format(ip))
def wait_nailgun_nodes(self):
"""Wait for cluster nodes online state in nailgun"""
cluster_id = self._storage['cluster_id']
fuel_web = self.manager.fuel_web
fuel_web.wait_cluster_nodes_get_online_state(cluster_id)
def wait_mcollective_nodes(self):
"""Wait for mcollective online status of cluster nodes"""
cluster_id = self._storage['cluster_id']
fuel_web = self.manager.fuel_web
wait(lambda: fuel_web.mcollective_nodes_online(cluster_id),
timeout=60 * 5, timeout_msg="Cluster nodes don't become available"
" via mcollective in allotted time.")
def wait_nailgun_available(self):
"""Check status for Nailgun"""
fuel_web = self.manager.fuel_web
_wait(fuel_web.get_nailgun_version,
timeout=60 * 20)
def compute_hard_restart(self):
"""Hard restart compute with Fuel Master node"""
fuel_web = self.manager.fuel_web
fuel_web.cold_restart_nodes(
[fuel_web.get_devops_node_by_nailgun_node(self.compute)],
wait_offline=False, wait_online=False, wait_after_destroy=5
)
def compute_warm_restart(self):
"""Warm restart of the compute with Fuel Master node"""
logger.debug('Reboot (warm restart) ip {0}'.format(self.compute['ip']))
with self.env.d_env.get_ssh_to_remote(self.compute['ip']) as remote:
remote.execute('/sbin/shutdown -r now')
def run_checkers(self):
"""Run set of checkers"""
self.wait_nailgun_available()
self.wait_mcollective_nodes()
self.wait_nailgun_nodes()

46
fuelweb_test/models/fuel_web_client.py
View File

@ -404,14 +404,16 @@ class FuelWebClient29(object):
checkers.fail_deploy(not_ready_transactions)
def wait_node_is_online(self, devops_node, timeout=60 * 5):
def wait_node_is_online(self, node, timeout=60 * 5):
# transform devops node to nailgun node
if isinstance(node, Node):
node = self.get_nailgun_node_by_devops_node(node)
logger.info(
'Wait for node {!r} online status'.format(devops_node.name))
wait(lambda: self.get_nailgun_node_by_devops_node(
devops_node)['online'],
'Wait for node {!r} online status'.format(node['name']))
wait(lambda: self.get_nailgun_node_online_status(node),
timeout=timeout,
timeout_msg='Node {!r} failed to become online'
''.format(devops_node.name))
''.format(node['name']))
def wait_node_is_offline(self, devops_node, timeout=60 * 5):
logger.info(
@ -1993,13 +1995,22 @@ class FuelWebClient29(object):
return ip_ranges, expected_ips
@logwrap
def get_nailgun_node_online_status(self, node):
return self.client.get_node_by_id(node['id'])['online']
def get_devops_node_online_status(self, devops_node):
return self.get_nailgun_node_online_status(
self.get_nailgun_node_by_devops_node(devops_node))
def warm_shutdown_nodes(self, devops_nodes, timeout=4 * 60):
logger.info('Shutting down (warm) nodes %s',
[n.name for n in devops_nodes])
for node in devops_nodes:
logger.debug('Shutdown node %s', node.name)
with self.get_ssh_for_node(node.name) as remote:
remote.check_call('/sbin/shutdown -Ph now')
nailgun_node = self.get_nailgun_node_by_devops_node(node)
self.ssh_manager.execute_on_remote(ip=nailgun_node['ip'],
cmd='/sbin/shutdown -Ph now')
for node in devops_nodes:
self.wait_node_is_offline(node, timeout=timeout)
@ -2159,6 +2170,13 @@ class FuelWebClient29(object):
for node in nodes:
self.wait_node_is_online(node, timeout=timeout)
@logwrap
def wait_cluster_nodes_get_online_state(self, cluster_id,
timeout=4 * 60):
self.wait_nodes_get_online_state(
self.client.list_cluster_nodes(cluster_id),
timeout=timeout)
@logwrap
def wait_mysql_galera_is_up(self, node_names, timeout=60 * 4):
def _get_galera_status(_remote):
@ -2187,6 +2205,20 @@ class FuelWebClient29(object):
node_name, _get_galera_status(remote)))
return True
@logwrap
def mcollective_nodes_online(self, cluster_id):
nodes_uids = set([str(n['id']) for n in
self.client.list_cluster_nodes(cluster_id)])
# 'mco find' returns '1' exit code if rabbitmq is not ready
out = self.ssh_manager.execute_on_remote(
ip=self.ssh_manager.admin_ip,
cmd='mco find', assert_ec_equal=[0, 1])['stdout_str']
ready_nodes_uids = set(out.split('\n'))
unavailable_nodes = nodes_uids - ready_nodes_uids
logger.debug('Nodes {0} are not reacheable via'
' mcollective'.format(unavailable_nodes))
return not unavailable_nodes
@logwrap
def wait_cinder_is_up(self, node_names):
logger.info("Waiting for all Cinder services up.")

4
fuelweb_test/models/nailgun_client.py
View File

@ -146,6 +146,10 @@ class NailgunClient(object):
def delete_cluster(self, cluster_id):
return self._delete(url="/clusters/{}/".format(cluster_id)).json()
@logwrap
def get_node_by_id(self, node_id):
return self._get(url="/nodes/{}".format(node_id)).json()
@logwrap
def update_node(self, node_id, data):
return self._put(