diff --git a/oswin_tempest_plugin/clients/__init__.py b/oswin_tempest_plugin/clients/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oswin_tempest_plugin/clients/wsman.py b/oswin_tempest_plugin/clients/wsman.py new file mode 100644 index 0000000..3ccb55d --- /dev/null +++ b/oswin_tempest_plugin/clients/wsman.py @@ -0,0 +1,60 @@ +# Copyright 2013 Cloudbase Solutions Srl +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +from winrm import protocol + +from oswin_tempest_plugin import exceptions + +LOG = logging.getLogger(__name__) + +protocol.Protocol.DEFAULT_TIMEOUT = "PT3600S" + + +def run_wsman_cmd(host, username, password, cmd, fail_on_error=False): + url = 'https://%s:5986/wsman' % host + LOG.debug('Connecting to: %s', host) + p = protocol.Protocol(endpoint=url, + transport='plaintext', + server_cert_validation='ignore', + username=username, + password=password) + + shell_id = p.open_shell() + LOG.debug('Running command on host %(host)s: %(cmd)s', + {'host': host, 'cmd': cmd}) + command_id = p.run_command(shell_id, cmd) + std_out, std_err, return_code = p.get_command_output(shell_id, command_id) + + p.cleanup_command(shell_id, command_id) + p.close_shell(shell_id) + + LOG.debug('Results from %(host)s: return_code: %(return_code)s, std_out: ' + '%(std_out)s, std_err: %(std_err)s', + {'host': host, 'return_code': return_code, 'std_out': std_out, + 'std_err': std_err}) + + if fail_on_error and return_code: + raise exceptions.WSManException( + cmd=cmd, host=host, return_code=return_code, + std_out=std_out, std_err=std_err) + + return (std_out, std_err, return_code) + + +def run_wsman_ps(host, username, password, cmd, fail_on_error=False): + cmd = ("powershell -NonInteractive -ExecutionPolicy RemoteSigned " + "-Command \"%s\"" % cmd) + return run_wsman_cmd(host, username, password, cmd, fail_on_error) diff --git a/oswin_tempest_plugin/config.py b/oswin_tempest_plugin/config.py index a543166..e43e776 100644 --- a/oswin_tempest_plugin/config.py +++ b/oswin_tempest_plugin/config.py @@ -36,6 +36,21 @@ HyperVGroup = [ cfg.StrOpt('gen2_image_ref', help="Valid Generation 2 VM VHDX image reference to be used " "in tests."), + cfg.BoolOpt('cluster_enabled', + default=False, + help="The compute nodes are joined into a Hyper-V Cluster."), + cfg.StrOpt('username', + help="The username of the Hyper-V hosts."), + cfg.StrOpt('password', + secret=True, + help='The password of the Hyper-V hosts.'), + cfg.IntOpt('failover_timeout', + default=120, + help='The maximum amount of time to wait for a failover to ' + 'occur.'), + cfg.IntOpt('failover_sleep_interval', + default=5, + help='The amount of time to wait between failover checks.'), ] diff --git a/oswin_tempest_plugin/exceptions.py b/oswin_tempest_plugin/exceptions.py index 5711beb..87343a0 100644 --- a/oswin_tempest_plugin/exceptions.py +++ b/oswin_tempest_plugin/exceptions.py @@ -19,3 +19,13 @@ from tempest.lib import exceptions class ResizeException(exceptions.TempestException): message = ("Server %(server_id)s failed to resize to the given " "flavor %(flavor)s") + + +class NotFoundException(exceptions.TempestException): + message = "Resource %(resource)s (%(res_type)s) was not found." + + +class WSManException(exceptions.TempestException): + message = ('Command "%(cmd)s" failed on host %(host)s failed with the ' + 'return code %(return_code)s. std_out: %(std_out)s, ' + 'std_err: %(std_err)s') diff --git a/oswin_tempest_plugin/tests/scenario/test_cluster.py b/oswin_tempest_plugin/tests/scenario/test_cluster.py new file mode 100644 index 0000000..1d6e3aa --- /dev/null +++ b/oswin_tempest_plugin/tests/scenario/test_cluster.py @@ -0,0 +1,158 @@ +# Copyright 2017 Cloudbase Solutions SRL +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time + +from oslo_log import log as logging +from tempest.lib import exceptions as lib_exc + +from oswin_tempest_plugin.clients import wsman +from oswin_tempest_plugin import config +from oswin_tempest_plugin import exceptions +from oswin_tempest_plugin.tests import test_base +from oswin_tempest_plugin.tests._mixins import migrate +from oswin_tempest_plugin.tests._mixins import resize + +CONF = config.CONF +LOG = logging.getLogger(__name__) + + +class HyperVClusterTest(test_base.TestBase, + migrate._MigrateMixin, + resize._ResizeMixin): + + """The test suite for the Hyper-V Cluster. + + This test suite will test the functionality of the Hyper-V Cluster Driver + in OpenStack. The tests will force a failover on its newly created + instance, and asserts the following: + + * the instance moves to another host. + * the nova instance's host is properly updated. + * the instance's network connection still works. + * different nova operations can be performed properly. + + This test suite relies on the fact that there are at least 2 compute nodes + available, that they are clustered, and have WSMan configured. + + The test suite contains the following tests: + + * test_check_clustered_vm + * test_check_migration + * test_check_resize + * test_check_resize_negative + """ + + _BIGGER_FLAVOR = {'disk': 1} + _BAD_FLAVOR = {'disk': -1} + + @classmethod + def skip_checks(cls): + super(HyperVClusterTest, cls).skip_checks() + + # check if the cluster Tests can be run. + conf_opts = ['cluster_enabled', 'username', 'password'] + for conf_opt in conf_opts: + if not getattr(CONF.hyperv, conf_opt): + msg = ('The config option "hyperv.%s" has not been set. ' + 'Skipping.' % conf_opt) + raise cls.skipException(msg) + + if not CONF.compute.min_compute_nodes >= 2: + msg = 'Expected at least 2 compute nodes.' + raise cls.skipException(msg) + + def _failover_server(self, server_name, host_ip): + """Triggers the failover for the given server on the given host.""" + + resource_name = "Virtual Machine %s" % server_name + cmd = "Test-ClusterResourceFailure -Name '%s'" % resource_name + # NOTE(claudiub): we issue the failover command twice, because on + # the first failure, the Hyper-V Cluster will prefer the current + # node, and will try to reactivate the VM on the it, and it will + # succeed. On the 2nd failure, the VM will failover to another + # node. Also, there needs to be a delay between commands, so the + # original failover has time to finish. + wsman.run_wsman_ps(host_ip, CONF.hyperv.username, + CONF.hyperv.password, cmd, True) + time.sleep(CONF.hyperv.failover_sleep_interval) + wsman.run_wsman_ps(host_ip, CONF.hyperv.username, + CONF.hyperv.password, cmd, True) + + def _wait_for_failover(self, server, original_host): + """Waits for the given server to failover to another host. + + :raises TimeoutException: if the given server did not failover to + another host within the configured "CONF.hyperv.failover_timeout" + interval. + """ + LOG.debug('Waiting for server %(server)s to failover from ' + 'compute node %(host)s', + dict(server=server['id'], host=original_host)) + + start_time = int(time.time()) + timeout = CONF.hyperv.failover_timeout + while True: + elapsed_time = int(time.time()) - start_time + admin_server = self._get_server_as_admin(server) + current_host = admin_server['OS-EXT-SRV-ATTR:host'] + if current_host != original_host: + LOG.debug('Server %(server)s failovered from compute node ' + '%(host)s in %(seconds)s seconds.', + dict(server=server['id'], host=original_host, + seconds=elapsed_time)) + return + + if elapsed_time >= timeout: + msg = ('Server %(server)s did not failover in the given ' + 'amount of time (%(timeout)s s).') + raise lib_exc.TimeoutException( + msg % dict(server=server['id'], timeout=timeout)) + + time.sleep(CONF.hyperv.failover_sleep_interval) + + def _get_hypervisor(self, hostname): + hypervisors = self.admin_hypervisor_client.list_hypervisors( + detail=True)['hypervisors'] + hypervisor = [h for h in hypervisors if + h['hypervisor_hostname'] == hostname] + + if not hypervisor: + raise exceptions.NotFoundException(resource=hostname, + res_type='hypervisor') + return hypervisor[0] + + def _get_server_as_admin(self, server): + # only admins have access to certain instance properties. + return self.admin_servers_client.show_server( + server['id'])['server'] + + def _create_server(self): + server_tuple = super(HyperVClusterTest, self)._create_server() + server = server_tuple.server + admin_server = self._get_server_as_admin(server) + + server_name = admin_server['OS-EXT-SRV-ATTR:instance_name'] + hostname = admin_server['OS-EXT-SRV-ATTR:host'] + host_ip = self._get_hypervisor(hostname)['host_ip'] + + self._failover_server(server_name, host_ip) + self._wait_for_failover(server, hostname) + + return server_tuple + + def test_clustered_vm(self): + server_tuple = self._create_server() + self._check_server_connectivity(server_tuple) diff --git a/oswin_tempest_plugin/tests/test_base.py b/oswin_tempest_plugin/tests/test_base.py index 24c58af..994fb28 100644 --- a/oswin_tempest_plugin/tests/test_base.py +++ b/oswin_tempest_plugin/tests/test_base.py @@ -76,6 +76,7 @@ class TestBase(tempest.test.BaseTestCase): cls.admin_servers_client = cls.os_admin.servers_client cls.admin_flavors_client = cls.os_admin.flavors_client cls.admin_migrations_client = cls.os_admin.migrations_client + cls.admin_hypervisor_client = cls.os_admin.hypervisor_client # Neutron network client cls.security_groups_client = ( diff --git a/requirements.txt b/requirements.txt index 1d18dd3..450deb6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ # process, which may cause wedges in the gate later. pbr>=2.0 # Apache-2.0 +pywinrm>=0.2.2 # MIT