From 782f677c13f567a9b41773b0db258defeb96cc37 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Wed, 11 Nov 2015 11:26:45 -0500 Subject: [PATCH] add test to ensure cloud is using minimum number of nodes This ensures that the the cloud in question is using some minimum number of nodes as specified by compute.min_compute_nodes. We then attempt to schedule to all those nodes directly with scheduler hints, and verify that the host_ids are all different on the booted nodes. This kind of test is needed when doing multinode rolling upgrade testing, because an extremely common failure is a change which makes the upgraded controller incompatible with a non upgraded worker. The worker will then stop communicating and effectively drop out of the cluster. All the other tests will pass because they just route around the damage. However we really do want to poke every node to make sure they all still work. Change-Id: I7d7230d100901179a1d1a0281aa8be990d00c02a --- tempest/config.py | 8 ++- tempest/scenario/test_server_multinode.py | 78 +++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tempest/scenario/test_server_multinode.py diff --git a/tempest/config.py b/tempest/config.py index 0cda0181b1..95553e1efc 100644 --- a/tempest/config.py +++ b/tempest/config.py @@ -330,7 +330,13 @@ ComputeGroup = [ help='Unallocated floating IP range, which will be used to ' 'test the floating IP bulk feature for CRUD operation. ' 'This block must not overlap an existing floating IP ' - 'pool.') + 'pool.'), + cfg.IntOpt('min_compute_nodes', + default=1, + help=('The minimum number of compute nodes expected. This will ' + 'be utilized by some multinode specific tests to ensure ' + 'that requests match the expected size of the cluster ' + 'you are testing with.')) ] compute_features_group = cfg.OptGroup(name='compute-feature-enabled', diff --git a/tempest/scenario/test_server_multinode.py b/tempest/scenario/test_server_multinode.py new file mode 100644 index 0000000000..1645081fc8 --- /dev/null +++ b/tempest/scenario/test_server_multinode.py @@ -0,0 +1,78 @@ +# Copyright 2012 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from oslo_log import log as logging + +from tempest import config +from tempest import exceptions +from tempest.scenario import manager +from tempest import test + +CONF = config.CONF + +LOG = logging.getLogger(__name__) + + +class TestServerMultinode(manager.ScenarioTest): + + """ + This is a set of tests specific to multinode testing. + + """ + credentials = ['primary', 'admin'] + + @test.idempotent_id('9cecbe35-b9d4-48da-a37e-7ce70aa43d30') + @test.attr(type='smoke') + @test.services('compute', 'network') + def test_schedule_to_all_nodes(self): + host_client = self.admin_manager.hosts_client + hosts = host_client.list_hosts()['hosts'] + hosts = [x for x in hosts if x['service'] == 'compute'] + + # ensure we have at least as many compute hosts as we expect + if len(hosts) < CONF.compute.min_compute_nodes: + raise exceptions.InvalidConfiguration( + "Host list %s is shorter than min_compute_nodes. " + "Did a compute worker not boot correctly?" % hosts) + + # create 1 compute for each node, up to the min_compute_nodes + # threshold (so that things don't get crazy if you have 1000 + # compute nodes but set min to 3). + servers = [] + + for host in hosts[:CONF.compute.min_compute_nodes]: + create_kwargs = { + 'scheduler_hints': {'force_hosts': host['host_name']} + } + + # by getting to active state here, this means this has + # landed on the host in question. + inst = self.create_server(image=CONF.compute.image_ref, + flavor=CONF.compute.flavor_ref, + create_kwargs=create_kwargs) + server = self.servers_client.show_server(inst['id'])['server'] + servers.append(server) + + # make sure we really have the number of servers we think we should + self.assertEqual( + len(servers), CONF.compute.min_compute_nodes, + "Incorrect number of servers built %s" % servers) + + # ensure that every server ended up on a different host + host_ids = [x['hostId'] for x in servers] + self.assertEqual( + len(set(host_ids)), len(servers), + "Incorrect number of distinct host_ids scheduled to %s" % servers)