summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorClaudiu Belu <cbelu@cloudbasesolutions.com>2017-06-26 06:59:57 -0700
committerClaudiu Belu <cbelu@cloudbasesolutions.com>2017-07-18 03:27:54 -0700
commit763348b367d3707153267b005f07bdbcf83fb8f1 (patch)
treec38f5f4fe06b362d885adc42d3a22f52662fd35a
parentbaa1ef318b7b55dfec5bd72e386b2d64670b666f (diff)
Adds Hyper-V Cluster scenario
Hyper-V VMs can be clustered, making them highly available. We can force a VM failover through WinRM, causing the VM to restart on another host. For this, the Hyper-V hosts must have WinRM enabled. A VM must have network connectivity after the failover, and operations (resize, migrate, etc.) must still succeed after failover. Adds the following config options: - cluster_enabled (default = False) - username - password - failover_timeout (default = 120 seconds) - failover_sleep_interval (default = 5 seconds) Adds HyperVClusterTest.
-rw-r--r--oswin_tempest_plugin/clients/__init__.py0
-rw-r--r--oswin_tempest_plugin/clients/wsman.py60
-rw-r--r--oswin_tempest_plugin/config.py15
-rw-r--r--oswin_tempest_plugin/exceptions.py10
-rw-r--r--oswin_tempest_plugin/tests/scenario/test_cluster.py158
-rw-r--r--oswin_tempest_plugin/tests/test_base.py1
-rw-r--r--requirements.txt1
7 files changed, 245 insertions, 0 deletions
diff --git a/oswin_tempest_plugin/clients/__init__.py b/oswin_tempest_plugin/clients/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/oswin_tempest_plugin/clients/__init__.py
diff --git a/oswin_tempest_plugin/clients/wsman.py b/oswin_tempest_plugin/clients/wsman.py
new file mode 100644
index 0000000..3ccb55d
--- /dev/null
+++ b/oswin_tempest_plugin/clients/wsman.py
@@ -0,0 +1,60 @@
1# Copyright 2013 Cloudbase Solutions Srl
2# All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16from oslo_log import log as logging
17from winrm import protocol
18
19from oswin_tempest_plugin import exceptions
20
21LOG = logging.getLogger(__name__)
22
23protocol.Protocol.DEFAULT_TIMEOUT = "PT3600S"
24
25
26def run_wsman_cmd(host, username, password, cmd, fail_on_error=False):
27 url = 'https://%s:5986/wsman' % host
28 LOG.debug('Connecting to: %s', host)
29 p = protocol.Protocol(endpoint=url,
30 transport='plaintext',
31 server_cert_validation='ignore',
32 username=username,
33 password=password)
34
35 shell_id = p.open_shell()
36 LOG.debug('Running command on host %(host)s: %(cmd)s',
37 {'host': host, 'cmd': cmd})
38 command_id = p.run_command(shell_id, cmd)
39 std_out, std_err, return_code = p.get_command_output(shell_id, command_id)
40
41 p.cleanup_command(shell_id, command_id)
42 p.close_shell(shell_id)
43
44 LOG.debug('Results from %(host)s: return_code: %(return_code)s, std_out: '
45 '%(std_out)s, std_err: %(std_err)s',
46 {'host': host, 'return_code': return_code, 'std_out': std_out,
47 'std_err': std_err})
48
49 if fail_on_error and return_code:
50 raise exceptions.WSManException(
51 cmd=cmd, host=host, return_code=return_code,
52 std_out=std_out, std_err=std_err)
53
54 return (std_out, std_err, return_code)
55
56
57def run_wsman_ps(host, username, password, cmd, fail_on_error=False):
58 cmd = ("powershell -NonInteractive -ExecutionPolicy RemoteSigned "
59 "-Command \"%s\"" % cmd)
60 return run_wsman_cmd(host, username, password, cmd, fail_on_error)
diff --git a/oswin_tempest_plugin/config.py b/oswin_tempest_plugin/config.py
index a543166..e43e776 100644
--- a/oswin_tempest_plugin/config.py
+++ b/oswin_tempest_plugin/config.py
@@ -36,6 +36,21 @@ HyperVGroup = [
36 cfg.StrOpt('gen2_image_ref', 36 cfg.StrOpt('gen2_image_ref',
37 help="Valid Generation 2 VM VHDX image reference to be used " 37 help="Valid Generation 2 VM VHDX image reference to be used "
38 "in tests."), 38 "in tests."),
39 cfg.BoolOpt('cluster_enabled',
40 default=False,
41 help="The compute nodes are joined into a Hyper-V Cluster."),
42 cfg.StrOpt('username',
43 help="The username of the Hyper-V hosts."),
44 cfg.StrOpt('password',
45 secret=True,
46 help='The password of the Hyper-V hosts.'),
47 cfg.IntOpt('failover_timeout',
48 default=120,
49 help='The maximum amount of time to wait for a failover to '
50 'occur.'),
51 cfg.IntOpt('failover_sleep_interval',
52 default=5,
53 help='The amount of time to wait between failover checks.'),
39] 54]
40 55
41 56
diff --git a/oswin_tempest_plugin/exceptions.py b/oswin_tempest_plugin/exceptions.py
index 5711beb..87343a0 100644
--- a/oswin_tempest_plugin/exceptions.py
+++ b/oswin_tempest_plugin/exceptions.py
@@ -19,3 +19,13 @@ from tempest.lib import exceptions
19class ResizeException(exceptions.TempestException): 19class ResizeException(exceptions.TempestException):
20 message = ("Server %(server_id)s failed to resize to the given " 20 message = ("Server %(server_id)s failed to resize to the given "
21 "flavor %(flavor)s") 21 "flavor %(flavor)s")
22
23
24class NotFoundException(exceptions.TempestException):
25 message = "Resource %(resource)s (%(res_type)s) was not found."
26
27
28class WSManException(exceptions.TempestException):
29 message = ('Command "%(cmd)s" failed on host %(host)s failed with the '
30 'return code %(return_code)s. std_out: %(std_out)s, '
31 'std_err: %(std_err)s')
diff --git a/oswin_tempest_plugin/tests/scenario/test_cluster.py b/oswin_tempest_plugin/tests/scenario/test_cluster.py
new file mode 100644
index 0000000..1d6e3aa
--- /dev/null
+++ b/oswin_tempest_plugin/tests/scenario/test_cluster.py
@@ -0,0 +1,158 @@
1# Copyright 2017 Cloudbase Solutions SRL
2# All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16import time
17
18from oslo_log import log as logging
19from tempest.lib import exceptions as lib_exc
20
21from oswin_tempest_plugin.clients import wsman
22from oswin_tempest_plugin import config
23from oswin_tempest_plugin import exceptions
24from oswin_tempest_plugin.tests import test_base
25from oswin_tempest_plugin.tests._mixins import migrate
26from oswin_tempest_plugin.tests._mixins import resize
27
28CONF = config.CONF
29LOG = logging.getLogger(__name__)
30
31
32class HyperVClusterTest(test_base.TestBase,
33 migrate._MigrateMixin,
34 resize._ResizeMixin):
35
36 """The test suite for the Hyper-V Cluster.
37
38 This test suite will test the functionality of the Hyper-V Cluster Driver
39 in OpenStack. The tests will force a failover on its newly created
40 instance, and asserts the following:
41
42 * the instance moves to another host.
43 * the nova instance's host is properly updated.
44 * the instance's network connection still works.
45 * different nova operations can be performed properly.
46
47 This test suite relies on the fact that there are at least 2 compute nodes
48 available, that they are clustered, and have WSMan configured.
49
50 The test suite contains the following tests:
51
52 * test_check_clustered_vm
53 * test_check_migration
54 * test_check_resize
55 * test_check_resize_negative
56 """
57
58 _BIGGER_FLAVOR = {'disk': 1}
59 _BAD_FLAVOR = {'disk': -1}
60
61 @classmethod
62 def skip_checks(cls):
63 super(HyperVClusterTest, cls).skip_checks()
64
65 # check if the cluster Tests can be run.
66 conf_opts = ['cluster_enabled', 'username', 'password']
67 for conf_opt in conf_opts:
68 if not getattr(CONF.hyperv, conf_opt):
69 msg = ('The config option "hyperv.%s" has not been set. '
70 'Skipping.' % conf_opt)
71 raise cls.skipException(msg)
72
73 if not CONF.compute.min_compute_nodes >= 2:
74 msg = 'Expected at least 2 compute nodes.'
75 raise cls.skipException(msg)
76
77 def _failover_server(self, server_name, host_ip):
78 """Triggers the failover for the given server on the given host."""
79
80 resource_name = "Virtual Machine %s" % server_name
81 cmd = "Test-ClusterResourceFailure -Name '%s'" % resource_name
82 # NOTE(claudiub): we issue the failover command twice, because on
83 # the first failure, the Hyper-V Cluster will prefer the current
84 # node, and will try to reactivate the VM on the it, and it will
85 # succeed. On the 2nd failure, the VM will failover to another
86 # node. Also, there needs to be a delay between commands, so the
87 # original failover has time to finish.
88 wsman.run_wsman_ps(host_ip, CONF.hyperv.username,
89 CONF.hyperv.password, cmd, True)
90 time.sleep(CONF.hyperv.failover_sleep_interval)
91 wsman.run_wsman_ps(host_ip, CONF.hyperv.username,
92 CONF.hyperv.password, cmd, True)
93
94 def _wait_for_failover(self, server, original_host):
95 """Waits for the given server to failover to another host.
96
97 :raises TimeoutException: if the given server did not failover to
98 another host within the configured "CONF.hyperv.failover_timeout"
99 interval.
100 """
101 LOG.debug('Waiting for server %(server)s to failover from '
102 'compute node %(host)s',
103 dict(server=server['id'], host=original_host))
104
105 start_time = int(time.time())
106 timeout = CONF.hyperv.failover_timeout
107 while True:
108 elapsed_time = int(time.time()) - start_time
109 admin_server = self._get_server_as_admin(server)
110 current_host = admin_server['OS-EXT-SRV-ATTR:host']
111 if current_host != original_host:
112 LOG.debug('Server %(server)s failovered from compute node '
113 '%(host)s in %(seconds)s seconds.',
114 dict(server=server['id'], host=original_host,
115 seconds=elapsed_time))
116 return
117
118 if elapsed_time >= timeout:
119 msg = ('Server %(server)s did not failover in the given '
120 'amount of time (%(timeout)s s).')
121 raise lib_exc.TimeoutException(
122 msg % dict(server=server['id'], timeout=timeout))
123
124 time.sleep(CONF.hyperv.failover_sleep_interval)
125
126 def _get_hypervisor(self, hostname):
127 hypervisors = self.admin_hypervisor_client.list_hypervisors(
128 detail=True)['hypervisors']
129 hypervisor = [h for h in hypervisors if
130 h['hypervisor_hostname'] == hostname]
131
132 if not hypervisor:
133 raise exceptions.NotFoundException(resource=hostname,
134 res_type='hypervisor')
135 return hypervisor[0]
136
137 def _get_server_as_admin(self, server):
138 # only admins have access to certain instance properties.
139 return self.admin_servers_client.show_server(
140 server['id'])['server']
141
142 def _create_server(self):
143 server_tuple = super(HyperVClusterTest, self)._create_server()
144 server = server_tuple.server
145 admin_server = self._get_server_as_admin(server)
146
147 server_name = admin_server['OS-EXT-SRV-ATTR:instance_name']
148 hostname = admin_server['OS-EXT-SRV-ATTR:host']
149 host_ip = self._get_hypervisor(hostname)['host_ip']
150
151 self._failover_server(server_name, host_ip)
152 self._wait_for_failover(server, hostname)
153
154 return server_tuple
155
156 def test_clustered_vm(self):
157 server_tuple = self._create_server()
158 self._check_server_connectivity(server_tuple)
diff --git a/oswin_tempest_plugin/tests/test_base.py b/oswin_tempest_plugin/tests/test_base.py
index 24c58af..994fb28 100644
--- a/oswin_tempest_plugin/tests/test_base.py
+++ b/oswin_tempest_plugin/tests/test_base.py
@@ -76,6 +76,7 @@ class TestBase(tempest.test.BaseTestCase):
76 cls.admin_servers_client = cls.os_admin.servers_client 76 cls.admin_servers_client = cls.os_admin.servers_client
77 cls.admin_flavors_client = cls.os_admin.flavors_client 77 cls.admin_flavors_client = cls.os_admin.flavors_client
78 cls.admin_migrations_client = cls.os_admin.migrations_client 78 cls.admin_migrations_client = cls.os_admin.migrations_client
79 cls.admin_hypervisor_client = cls.os_admin.hypervisor_client
79 80
80 # Neutron network client 81 # Neutron network client
81 cls.security_groups_client = ( 82 cls.security_groups_client = (
diff --git a/requirements.txt b/requirements.txt
index 1d18dd3..450deb6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,4 @@
3# process, which may cause wedges in the gate later. 3# process, which may cause wedges in the gate later.
4 4
5pbr>=2.0 # Apache-2.0 5pbr>=2.0 # Apache-2.0
6pywinrm>=0.2.2 # MIT