605 lines
22 KiB
Python
605 lines
22 KiB
Python
# Copyright 2015-2016 Mirantis, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE_2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from __future__ import division
|
|
|
|
import time
|
|
import itertools
|
|
|
|
|
|
from proboscis import SkipTest
|
|
from proboscis.asserts import assert_equal
|
|
from proboscis.asserts import assert_true
|
|
# pylint: disable=redefined-builtin
|
|
# noinspection PyUnresolvedReferences
|
|
from six.moves import xrange
|
|
# pylint: enable=redefined-builtin
|
|
|
|
from fuelweb_test.helpers import checkers
|
|
from fuelweb_test.helpers.utils import TimeStat
|
|
from fuelweb_test import settings
|
|
|
|
from system_test import logger
|
|
from system_test import action
|
|
from system_test import nested_action
|
|
from system_test import deferred_decorator
|
|
|
|
from system_test.actions.ostf_actions import HealthCheckActions
|
|
from system_test.actions.plugins_actions import PluginsActions
|
|
from system_test.actions.vcenter_actions import VMwareActions
|
|
|
|
from system_test.core.discover import load_yaml
|
|
from system_test.helpers.decorators import make_snapshot_if_step_fail
|
|
|
|
|
|
# pylint: disable=no-member
|
|
# noinspection PyUnresolvedReferences
|
|
class PrepareActions(object):
|
|
"""Base class with prepare actions
|
|
|
|
_start_case - runned before test case start
|
|
_finish_case - runned after test case finish
|
|
setup_master - setup master node in environment
|
|
config_release - preconfig releases if it needs
|
|
make_slaves - boot slaves and snapshot environment with bootstrapped slaves
|
|
revert_slaves - revert environment with bootstrapped slaves
|
|
|
|
"""
|
|
def __init__(self):
|
|
self.full_config = None
|
|
self.env_config = None
|
|
self.env_settings = None
|
|
self.config_name = None
|
|
self._devops_config = None
|
|
self._start_time = 0
|
|
|
|
def _load_config(self):
|
|
config = load_yaml(self.config_file)
|
|
self.full_config = config
|
|
self.env_config = config[
|
|
'template']['cluster_template']
|
|
self.env_settings = config[
|
|
'template']['cluster_template']['settings']
|
|
self.config_name = config['template']['name']
|
|
|
|
if 'devops_settings' in config['template']:
|
|
self._devops_config = config
|
|
|
|
def _start_case(self):
|
|
"""Start test case"""
|
|
self._load_config()
|
|
class_doc = getattr(self, "__doc__", self.__class__.__name__)
|
|
name = class_doc.splitlines()[0]
|
|
class_scenario = class_doc.splitlines()[1:]
|
|
start_case = "[ START {} ]".format(name)
|
|
header = "<<< {:=^142} >>>".format(start_case)
|
|
indent = ' ' * 4
|
|
scenario = '\n'.join(class_scenario)
|
|
logger.info("\n{header}\n\n"
|
|
"{indent}Configuration: {config}\n"
|
|
"\n{scenario}".format(
|
|
header=header,
|
|
indent=indent,
|
|
config=self.config_name,
|
|
scenario=scenario))
|
|
self._start_time = time.time()
|
|
|
|
def _finish_case(self):
|
|
"""Finish test case"""
|
|
case_time = time.time() - self._start_time
|
|
minutes = case_time // 60
|
|
# pylint: disable=round-builtin
|
|
seconds = int(round(case_time)) % 60
|
|
# pylint: enable=round-builtin
|
|
name = getattr(self, "__doc__",
|
|
self.__class__.__name__).splitlines()[0]
|
|
finish_case = "[ FINISH {} CASE TOOK {} min {} sec ]".format(
|
|
name,
|
|
minutes,
|
|
seconds)
|
|
footer = "<<< {:=^142} >>>".format(finish_case)
|
|
logger.info("\n{footer}\n".format(footer=footer))
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def setup_master(self):
|
|
"""Setup master node"""
|
|
self.check_run("empty")
|
|
with TimeStat("setup_environment", is_uniq=True):
|
|
if list(self.env.d_env.get_nodes(role='fuel_master')):
|
|
self.env.setup_environment()
|
|
self.fuel_post_install_actions()
|
|
|
|
elif list(self.env.d_env.get_nodes(role='centos_master')):
|
|
# need to use centos_master.yaml devops template
|
|
hostname = ''.join((settings.FUEL_MASTER_HOSTNAME,
|
|
settings.DNS_SUFFIX))
|
|
self.centos_setup_fuel(hostname)
|
|
|
|
else:
|
|
raise SkipTest(
|
|
"No Fuel master nodes found!")
|
|
|
|
self.env.make_snapshot("empty", is_make=True)
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def config_release(self):
|
|
"""Configuration releases"""
|
|
self.check_run("ready")
|
|
self.env.revert_snapshot("empty", skip_timesync=True)
|
|
|
|
self.fuel_web.get_nailgun_version()
|
|
self.fuel_web.change_default_network_settings()
|
|
|
|
if (settings.REPLACE_DEFAULT_REPOS and
|
|
settings.REPLACE_DEFAULT_REPOS_ONLY_ONCE):
|
|
self.fuel_web.replace_default_repos()
|
|
|
|
self.env.make_snapshot("ready", is_make=True)
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def make_slaves(self):
|
|
"""Bootstrap slave and make snapshot
|
|
|
|
Use slaves parameter from case section
|
|
|
|
"""
|
|
slaves = int(self.full_config['template']['slaves'])
|
|
snapshot_name = "ready_with_{}_slaves".format(slaves)
|
|
self.check_run(snapshot_name)
|
|
self.env.revert_snapshot("ready", skip_timesync=True)
|
|
logger.info("Bootstrap {} nodes".format(slaves))
|
|
self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[:slaves],
|
|
skip_timesync=True)
|
|
self.env.make_snapshot(snapshot_name, is_make=True)
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def revert_slaves(self):
|
|
"""Revert bootstrapped nodes
|
|
|
|
Skip if snapshot with cluster exists
|
|
|
|
"""
|
|
self.check_run(self.env_config['name'])
|
|
slaves = int(self.full_config['template']['slaves'])
|
|
snapshot_name = "ready_with_{}_slaves".format(slaves)
|
|
self.env.revert_snapshot(snapshot_name)
|
|
|
|
# noinspection PyMethodParameters
|
|
@nested_action
|
|
def prepare_admin_node_with_slaves():
|
|
"""Combine preparation steps in alias"""
|
|
return [
|
|
'setup_master',
|
|
'config_release',
|
|
'make_slaves',
|
|
'revert_slaves',
|
|
]
|
|
|
|
|
|
# noinspection PyUnresolvedReferences
|
|
class BaseActions(PrepareActions, HealthCheckActions, PluginsActions,
|
|
VMwareActions):
|
|
"""Basic actions for acceptance cases
|
|
|
|
For choosing action order use actions_order variable, set list of actions
|
|
order
|
|
|
|
Actions:
|
|
create_env - create and configure environment
|
|
add_nodes - add nodes to environment
|
|
deploy_cluster - deploy en environment
|
|
network_check - run network check
|
|
reset_cluster - reset an environment
|
|
delete_cluster - delete en environment
|
|
stop_deploy - stop deploying of environment
|
|
|
|
"""
|
|
|
|
est_duration = None
|
|
base_group = None
|
|
actions_order = None
|
|
cluster_id = None
|
|
scale_step = 0
|
|
power_step = 0
|
|
|
|
def _add_node(self, nodes_list):
|
|
"""Add nodes to Environment"""
|
|
logger.info("Add nodes to env {}".format(self.cluster_id))
|
|
names = "slave-{:02}"
|
|
slaves = int(self.full_config['template']['slaves'])
|
|
num = iter(xrange(1, slaves + 1))
|
|
nodes = {}
|
|
for new in nodes_list:
|
|
for _ in xrange(new['count']):
|
|
name = names.format(next(num))
|
|
while name in self.assigned_slaves:
|
|
name = names.format(next(num))
|
|
|
|
self.assigned_slaves.add(name)
|
|
nodes[name] = new['roles']
|
|
logger.info("Set roles {} to node {}".format(new['roles'],
|
|
name))
|
|
self.fuel_web.update_nodes(self.cluster_id, nodes)
|
|
|
|
def _del_node(self, nodes_list):
|
|
"""Delete nodes from Environment"""
|
|
logger.info("Delete nodes from env {}".format(self.cluster_id))
|
|
nodes = {}
|
|
|
|
for node in nodes_list:
|
|
cluster_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
|
|
self.cluster_id, node['roles'])
|
|
for i in xrange(node['count']):
|
|
dnode = self.fuel_web.get_devops_node_by_nailgun_node(
|
|
cluster_nodes[i])
|
|
self.assigned_slaves.remove(dnode.name)
|
|
|
|
nodes[dnode.name] = node['roles']
|
|
logger.info("Delete node {} with role {}".format(
|
|
dnode.name, node['roles']))
|
|
|
|
self.fuel_web.update_nodes(self.cluster_id, nodes, False, True)
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def create_env(self):
|
|
"""Create Fuel Environment
|
|
|
|
For configure Environment use environment-config section in config file
|
|
|
|
Skip action if we have snapshot with Environment name
|
|
|
|
"""
|
|
self.check_run(self.env_config['name'])
|
|
|
|
logger.info("Create env {}".format(
|
|
self.env_config['name']))
|
|
cluster_settings = {
|
|
"murano": self.env_settings['components'].get('murano', False),
|
|
"sahara": self.env_settings['components'].get('sahara', False),
|
|
"ceilometer": self.env_settings['components'].get('ceilometer',
|
|
False),
|
|
"ironic": self.env_settings['components'].get('ironic', False),
|
|
"user": self.env_config.get("user", "admin"),
|
|
"password": self.env_config.get("password", "admin"),
|
|
"tenant": self.env_config.get("tenant", "admin"),
|
|
"volumes_lvm": self.env_settings['storages'].get("volume-lvm",
|
|
False),
|
|
"volumes_ceph": self.env_settings['storages'].get("volume-ceph",
|
|
False),
|
|
"images_ceph": self.env_settings['storages'].get("image-ceph",
|
|
False),
|
|
"ephemeral_ceph": self.env_settings['storages'].get(
|
|
"ephemeral-ceph", False),
|
|
"objects_ceph": self.env_settings['storages'].get("rados-ceph",
|
|
False),
|
|
"osd_pool_size": str(self.env_settings['storages'].get(
|
|
"replica-ceph", 2)),
|
|
"net_provider": self.env_config['network'].get('provider',
|
|
'neutron'),
|
|
"net_segment_type": self.env_config['network'].get('segment-type',
|
|
'vlan'),
|
|
"assign_to_all_nodes": self.env_config['network'].get(
|
|
'pubip-to-all',
|
|
False),
|
|
"neutron_l3_ha": self.env_config['network'].get(
|
|
'neutron-l3-ha', False),
|
|
"neutron_dvr": self.env_config['network'].get(
|
|
'neutron-dvr', False),
|
|
"neutron_l2_pop": self.env_config['network'].get(
|
|
'neutron-l2-pop', False),
|
|
"neutron_qos": self.env_config['network'].get(
|
|
'neutron-qos', False),
|
|
}
|
|
|
|
self.cluster_id = self.fuel_web.create_cluster(
|
|
name=self.env_config['name'],
|
|
mode=settings.DEPLOYMENT_MODE,
|
|
release_name=settings.OPENSTACK_RELEASE_UBUNTU
|
|
if self.env_config['release'] == 'ubuntu'
|
|
else settings.OPENSTACK_RELEASE,
|
|
settings=cluster_settings)
|
|
|
|
logger.info("Cluster created with ID:{}".format(self.cluster_id))
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def add_nodes(self):
|
|
"""Add nodes to environment
|
|
|
|
Used sub-section nodes in environment-config section
|
|
|
|
Skip action if cluster doesn't exist
|
|
|
|
"""
|
|
if self.cluster_id is None:
|
|
raise SkipTest(
|
|
"The cluster_id is not specified, can not add node to cluster"
|
|
)
|
|
|
|
self._add_node(self.env_config['nodes'])
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def deploy_cluster(self):
|
|
"""Deploy environment
|
|
|
|
Skip action if cluster doesn't exist
|
|
|
|
"""
|
|
if self.cluster_id is None:
|
|
raise SkipTest(
|
|
"The cluster_id is not specified, can not run the deploy"
|
|
)
|
|
|
|
self.fuel_web.deploy_cluster_wait(self.cluster_id)
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def stop_on_deploy(self):
|
|
"""Stop environment deploying and wait while slave bacame online"""
|
|
if self.cluster_id is None:
|
|
raise SkipTest(
|
|
"The cluster_id is not specified, can not stop the deploy"
|
|
)
|
|
|
|
cluster_id = self.cluster_id
|
|
self.fuel_web.deploy_cluster_wait_progress(cluster_id, progress=60)
|
|
self.fuel_web.stop_deployment_wait(cluster_id)
|
|
self.fuel_web.wait_nodes_get_online_state(
|
|
self.env.d_env.get_nodes(name__in=list(self.assigned_slaves)),
|
|
timeout=10 * 60)
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def network_check(self):
|
|
"""Run network checker
|
|
|
|
Skip action if cluster doesn't exist
|
|
|
|
"""
|
|
if self.cluster_id is None:
|
|
raise SkipTest("The cluster_id is not specified, "
|
|
"can not run network verification")
|
|
|
|
self.fuel_web.verify_network(self.cluster_id)
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def save_load_environment(self):
|
|
"""Load existent environment from snapshot or save it"""
|
|
env_name = self.env_config['name']
|
|
if self.cluster_id is None:
|
|
logger.info("Revert Environment from "
|
|
"snapshot({})".format(env_name))
|
|
assert_true(self.env.d_env.has_snapshot(env_name))
|
|
self.env.revert_snapshot(env_name)
|
|
self.cluster_id = self.fuel_web.client.get_cluster_id(env_name)
|
|
logger.info("Cluster with ID:{} reverted".format(self.cluster_id))
|
|
else:
|
|
logger.info("Make snapshot of Environment '{}' ID:{}".format(
|
|
env_name, self.cluster_id))
|
|
self.env.make_snapshot(env_name, is_make=True)
|
|
self.env.resume_environment()
|
|
self.env.sync_time()
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def check_haproxy(self):
|
|
"""HAProxy backend checking"""
|
|
controller_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
|
|
self.cluster_id, ['controller'])
|
|
|
|
for node in controller_nodes:
|
|
logger.info("Check all HAProxy backends on {}".format(
|
|
node['meta']['system']['fqdn']))
|
|
haproxy_status = checkers.check_haproxy_backend(node['ip'])
|
|
assert_equal(haproxy_status['exit_code'], 1,
|
|
"HAProxy backends are DOWN. {0}".format(
|
|
haproxy_status))
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def scale_node(self):
|
|
"""Scale node in cluster
|
|
|
|
For add nodes with role use scale_nodes in yaml with action add in
|
|
step::
|
|
|
|
scale_nodes:
|
|
- - roles:
|
|
- controller
|
|
count: 2
|
|
action: add
|
|
|
|
For remove nodes with role use scale_nodes in yaml with action delete
|
|
in step::
|
|
|
|
scale_nodes:
|
|
- - roles:
|
|
- controller
|
|
count: 2
|
|
action: delete
|
|
|
|
Step may contain add and remove action together::
|
|
|
|
scale_nodes:
|
|
- - roles:
|
|
- compute
|
|
count: 2
|
|
action: add
|
|
- - roles:
|
|
- ceph-osd
|
|
count: 1
|
|
action: delete
|
|
|
|
"""
|
|
step_config = self.env_config['scale_nodes'][self.scale_step]
|
|
for node in step_config:
|
|
if node['action'] == 'add':
|
|
self._add_node([node])
|
|
if node.get('vmware_vcenter'):
|
|
nova_computes = node['vmware_vcenter']['nova-compute']
|
|
dvs_settings = node['vmware_dvs']
|
|
self.add_vmware_nova_compute(nova_computes)
|
|
self.update_dvs_plugin_settings(dvs_settings)
|
|
elif node['action'] == 'delete':
|
|
self._del_node([node])
|
|
if 'compute-vmware' in node['roles']:
|
|
self.del_vmware_nova_compute()
|
|
else:
|
|
logger.error("Unknown scale action: {}".format(node['action']))
|
|
self.scale_step += 1
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def manage_nodes_power(self):
|
|
"""Manage power of node
|
|
|
|
To power off node with role use manage_nodes_power in yaml with action
|
|
power_off in step::
|
|
|
|
manage_nodes_power:
|
|
- - roles:
|
|
- controller
|
|
node_number: 0
|
|
action: power_off
|
|
|
|
To power on node with role use manage_nodes_power in yaml with action
|
|
power_on in step::
|
|
|
|
manage_nodes_power:
|
|
- - roles:
|
|
- controller
|
|
node_number: 0
|
|
action: power_on
|
|
|
|
To restart node with role use manage_nodes_power in yaml with action
|
|
warm_restart or cold_restart in step::
|
|
|
|
manage_nodes_power:
|
|
- - roles:
|
|
- controller
|
|
node_number: 0
|
|
action: warm_restart
|
|
|
|
Example of cold restarting two different nodes with the same role::
|
|
|
|
manage_nodes_power:
|
|
- - roles:
|
|
- controller
|
|
node_number: 0
|
|
action: cold_restart
|
|
- - roles:
|
|
- controller
|
|
node_number: 1
|
|
action: cold_restart
|
|
|
|
"""
|
|
power_actions = {
|
|
'power_off_warm': self.fuel_web.warm_shutdown_nodes,
|
|
'power_on_warm': self.fuel_web.warm_start_nodes,
|
|
'reboot_warm': self.fuel_web.warm_restart_nodes,
|
|
'reboot_cold': self.fuel_web.cold_restart_nodes
|
|
}
|
|
|
|
step_config = self.env_config['manage_nodes_power'][self.power_step]
|
|
for node in step_config:
|
|
power_action = power_actions.get(node['action'], None)
|
|
node_number = node['node_number']
|
|
if power_action:
|
|
ng_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
|
|
cluster_id=self.cluster_id, roles=[node['roles'][0]])
|
|
|
|
dev_node = self.fuel_web.get_devops_node_by_nailgun_fqdn(
|
|
ng_nodes[node_number]['fqdn'])
|
|
|
|
power_action([dev_node])
|
|
else:
|
|
logger.error("Unknown power switch action: "
|
|
"{}".format(node['action']))
|
|
self.power_step += 1
|
|
|
|
def add_vmware_nova_compute(self, nova_computes):
|
|
vmware_attr = \
|
|
self.fuel_web.client.get_cluster_vmware_attributes(self.cluster_id)
|
|
vcenter_data = vmware_attr['editable']['value']['availability_zones'][
|
|
0]["nova_computes"]
|
|
|
|
comp_vmware_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
|
|
self.cluster_id, ['compute-vmware'], role_status='pending_roles')
|
|
|
|
for instance in nova_computes:
|
|
cluster_name = instance['cluster']
|
|
srv_name = instance['srv_name']
|
|
datastore = instance['datastore']
|
|
if instance['target_node'] == 'compute-vmware':
|
|
node = comp_vmware_nodes.pop()
|
|
target_node = node['hostname']
|
|
else:
|
|
target_node = instance['target_node']
|
|
|
|
vcenter_data.append(
|
|
{"vsphere_cluster": cluster_name,
|
|
"service_name": srv_name,
|
|
"datastore_regex": datastore,
|
|
"target_node": {
|
|
"current": {"id": target_node,
|
|
"label": target_node},
|
|
"options": [{"id": target_node,
|
|
"label": target_node}, ]},
|
|
}
|
|
)
|
|
|
|
logger.debug("Try to update cluster with next "
|
|
"vmware_attributes {0}".format(vmware_attr))
|
|
self.fuel_web.client.update_cluster_vmware_attributes(
|
|
self.cluster_id, vmware_attr)
|
|
|
|
def del_vmware_nova_compute(self):
|
|
vmware_attr = \
|
|
self.fuel_web.client.get_cluster_vmware_attributes(self.cluster_id)
|
|
vcenter_data = vmware_attr['editable']['value']['availability_zones'][
|
|
0]["nova_computes"]
|
|
|
|
comp_vmware_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
|
|
self.cluster_id, ['compute-vmware'])
|
|
comp_vmware_nodes = [node for node in comp_vmware_nodes if
|
|
node['pending_deletion'] is True]
|
|
|
|
for node, nova_comp in itertools.product(comp_vmware_nodes,
|
|
vcenter_data):
|
|
if node['hostname'] == nova_comp['target_node']['current']['id']:
|
|
vcenter_data.remove(nova_comp)
|
|
self.fuel_web.client.update_cluster_vmware_attributes(self.cluster_id,
|
|
vmware_attr)
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def reset_cluster(self):
|
|
"""Reset environment"""
|
|
cluster_id = self.cluster_id
|
|
self.fuel_web.stop_reset_env_wait(cluster_id)
|
|
|
|
@deferred_decorator([make_snapshot_if_step_fail])
|
|
@action
|
|
def delete_cluster(self):
|
|
"""Delete environment"""
|
|
cluster_id = self.cluster_id
|
|
self.fuel_web.delete_env_wait(cluster_id)
|