performa
/
shaker
Code Issues Proposed changes

Merge "Enhance Shaker to use flavor metadata matching with host aggregates"

This commit is contained in:
Zuul 2018-10-25 08:59:26 +00:00 committed by Gerrit Code Review
parent 1e7a67d19e f4276baa19
commit b300583ebb
5 changed files with 231 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
doc/source/usage.rst
View File

@ -87,7 +87,7 @@ that allow control the scheduling precisely:
* ``single_room`` - 1 instance per compute node
* ``double_room`` - 2 instances per compute node
* ``density: N`` - the multiplier for number of instances per compute node
* ``compute_nodes: N`` - how many compute nodes should be used (by default Shaker use all of them)
* ``compute_nodes: N`` - how many compute nodes should be used (by default Shaker use all of them \*see note below)
* ``zones: [Z1, Z2]`` - list of Nova availability zones to use
Examples:
@ -100,6 +100,12 @@ As result of deployment the set of agents is produced. For networking testing th
agents in ``master`` and ``slave`` roles. Master agents are controlled by ``shaker`` tool and execute commands.
Slaves are used as back-ends and do not receive any commands directly.
\*If a flavor is chosen, which has aggregate_instance_extra_specs metadata set to match a host aggregate, Shaker will only use matching computes for compute_nodes calculations.
If no aggregate_instance_extra_specs is set on a flavor Shaker will use all computes by default.
For example if we have 10 computes in a host aggregate with metadata special_hardware=true and use a flavor with
aggregate_instance_extra_specs:special_hardware=true Shaker will only take into account the 10 matching computes, and by default try to use all of them
Execution
^^^^^^^^^

3
shaker/engine/deploy.py
View File

@ -251,7 +251,8 @@ class Deployment(object):
def _get_compute_nodes(self, accommodation):
try:
return nova.get_available_compute_nodes(self.openstack_client.nova)
return nova.get_available_compute_nodes(self.openstack_client.nova,
self.flavor_name)
except nova.ForbiddenException:
# user has no permissions to list compute nodes
LOG.info('OpenStack user does not have permission to list compute '

43
shaker/openstack/clients/nova.py
View File

@ -20,7 +20,6 @@ import time
from novaclient import client as nova_client_pkg
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
@ -28,11 +27,45 @@ class ForbiddenException(nova_client_pkg.exceptions.Forbidden):
pass
def get_available_compute_nodes(nova_client):
def get_available_compute_nodes(nova_client, flavor_name):
try:
return [dict(host=svc.host, zone=svc.zone)
for svc in nova_client.services.list(binary='nova-compute')
if svc.state == 'up' and svc.status == 'enabled']
host_list = [dict(host=svc.host, zone=svc.zone)
for svc in
nova_client.services.list(binary='nova-compute')
if svc.state == 'up' and svc.status == 'enabled']
# If the flavor has aggregate_instance_extra_specs set then filter
# host_list to pick only the hosts matching the chosen flavor.
flavor = get_flavor(nova_client, flavor_name)
if flavor is not None:
extra_specs = flavor.get_keys()
for item in extra_specs:
if "aggregate_instance_extra_specs" in item:
LOG.debug('Flavor contains %s, using compute node '
'filtering', extra_specs)
# getting the extra spec seting for flavor in the
# standard format of extra_spec:value
extra_spec = item.split(":")[1]
extra_spec_value = extra_specs.get(item)
# create a set of aggregate host which match
agg_hosts = set(itertools.chain(
*[agg.hosts for agg in
nova_client.aggregates.list() if
agg.metadata.get(extra_spec) == extra_spec_value]))
# update list of available hosts with
# host_aggregate cross-check
host_list = [elem for elem in host_list if
elem['host'] in agg_hosts]
LOG.debug('Available compute nodes: %s ', host_list)
return host_list
except nova_client_pkg.exceptions.Forbidden:
msg = 'Forbidden to get list of compute nodes'
raise ForbiddenException(msg)

61
shaker/tests/fakes.py Normal file
View File

@ -0,0 +1,61 @@
# Copyright (c) 2015 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import uuid
DEFAULT_TIMESTAMP = '2018-10-22T00:00:00.000000'
class FakeNovaServiceList(object):
def __init__(self, status='enabled', binary='nova-compute', zone='nova',
state='up', updated_at=DEFAULT_TIMESTAMP,
host='host-1', disabled=None):
self.status = status
self.binary = binary
self.zone = zone
self.state = state
self.updated_at = updated_at
self.host = host
self.disabled = disabled
self.id = uuid.uuid4()
class FakeNovaFlavorList(object):
def __init__(self, name='test-flavor', ram=512, vcpus=1, disk=20,
extra_specs={}):
self.name = name
self.ram = ram
self.vcpus = vcpus
self.disk = disk
self.id = uuid.uuid4()
self.extra_specs = extra_specs
def get_keys(self):
return self.extra_specs
class FakeNovaAggregateList(object):
def __init__(self, name='test-aggregate', availability_zone='nova',
deleted=False, created_at=DEFAULT_TIMESTAMP, updated_at='',
deleted_at='', hosts=[], metadata={}):
self.name = name
self.availability_zone = availability_zone
self.deleted = deleted
self.created_at = created_at
self.updated_at = updated_at
self.deleted_at = deleted_at
self.hosts = hosts
self.metadata = metadata
self.id = uuid.uuid4()

129
shaker/tests/test_deploy.py
View File

@ -24,6 +24,7 @@ from oslo_config import fixture as config_fixture_pkg
from shaker.engine import config
from shaker.engine import deploy
from shaker.openstack.clients import nova
from shaker.tests import fakes
ZONE = 'zone'
@ -33,7 +34,6 @@ def nodes_helper(*nodes):
class TestDeploy(testtools.TestCase):
def setUp(self):
super(TestDeploy, self).setUp()
@ -551,8 +551,8 @@ class TestDeploy(testtools.TestCase):
expected = {
'agent': {'id': 'agent', 'mode': 'alone'}
}
agents = deployment.deploy({'agents':
[{'id': 'agent', 'mode': 'alone'}]})
agents = deployment.deploy(
{'agents': [{'id': 'agent', 'mode': 'alone'}]})
self.assertEqual(expected, agents)
@ -562,12 +562,127 @@ class TestDeploy(testtools.TestCase):
self.assertRaises(deploy.DeploymentException,
deployment.deploy, {'template': 'foo'})
@mock.patch('shaker.openstack.clients.openstack.OpenStackClient')
def test_get_compute_nodes_flavor_no_extra_specs(self,
nova_client_mock):
# setup fake nova api service list response
compute_host_1 = fakes.FakeNovaServiceList(host='host-1')
compute_host_2 = fakes.FakeNovaServiceList(host='host-2')
compute_host_3 = fakes.FakeNovaServiceList(host='host-3')
nova_client_mock.nova.services.list.return_value = [compute_host_1,
compute_host_2,
compute_host_3]
# setup fake nova api flavor list response
flavor_no_exta_specs = fakes.FakeNovaFlavorList(
name='flavor_no_exta_specs')
nova_client_mock.nova.flavors.list.return_value = [
flavor_no_exta_specs]
deployment = deploy.Deployment()
deployment.flavor_name = 'flavor_no_exta_specs'
deployment.openstack_client = nova_client_mock
accommodation = {'compute_nodes': 3}
expected = [{'host': 'host-1', 'zone': 'nova'},
{'host': 'host-2', 'zone': 'nova'},
{'host': 'host-3', 'zone': 'nova'}]
observed = deployment._get_compute_nodes(accommodation)
self.assertEqual(expected, observed)
@mock.patch('shaker.openstack.clients.openstack.OpenStackClient')
def test_get_compute_nodes_flavor_extra_specs_no_match(self,
nova_client_mock):
# setup fake nova api service list response
compute_host_1 = fakes.FakeNovaServiceList(host='host-1')
compute_host_2 = fakes.FakeNovaServiceList(host='host-2')
compute_host_3 = fakes.FakeNovaServiceList(host='host-3')
nova_client_mock.nova.services.list.return_value = [compute_host_1,
compute_host_2,
compute_host_3]
# setup fake nova api flavor list response
flavor_with_extra_specs = fakes.FakeNovaFlavorList(
name='flavor_with_extra_specs',
extra_specs={'aggregate_instance_extra_specs:other_hw': 'false'})
nova_client_mock.nova.flavors.list.return_value = [
flavor_with_extra_specs]
# setup fake nova api aggregate list response
agg_host_1 = fakes.FakeNovaAggregateList(hosts=['host-1'], metadata={
'special_hw': 'true'})
agg_host_2 = fakes.FakeNovaAggregateList(hosts=['host-2'])
agg_host_3 = fakes.FakeNovaAggregateList(hosts=['host-3'])
nova_client_mock.nova.aggregates.list.return_value = [agg_host_1,
agg_host_2,
agg_host_3]
deployment = deploy.Deployment()
deployment.flavor_name = 'flavor_with_extra_specs'
deployment.openstack_client = nova_client_mock
accommodation = {'compute_nodes': 3}
expected = []
observed = deployment._get_compute_nodes(accommodation)
self.assertEqual(expected, observed)
@mock.patch('shaker.openstack.clients.openstack.OpenStackClient')
def test_get_compute_nodes_flavor_extra_specs_with_match(self,
nova_client_mock):
# setup fake nova api service list response
compute_host_1 = fakes.FakeNovaServiceList(host='host-1')
compute_host_2 = fakes.FakeNovaServiceList(host='host-2')
compute_host_3 = fakes.FakeNovaServiceList(host='host-3')
nova_client_mock.nova.services.list.return_value = [compute_host_1,
compute_host_2,
compute_host_3]
# setup fake nova api flavor list response
flavor_with_extra_specs = fakes.FakeNovaFlavorList(
name='flavor_with_extra_specs',
extra_specs={'aggregate_instance_extra_specs:special_hw': 'true'})
nova_client_mock.nova.flavors.list.return_value = [
flavor_with_extra_specs]
# setup fake nova api aggregate list response
agg_host_1 = fakes.FakeNovaAggregateList(hosts=['host-1'])
agg_host_2 = fakes.FakeNovaAggregateList(hosts=['host-2'], metadata={
'special_hw': 'true'})
agg_host_3 = fakes.FakeNovaAggregateList(hosts=['host-3'])
nova_client_mock.nova.aggregates.list.return_value = [agg_host_1,
agg_host_2,
agg_host_3]
deployment = deploy.Deployment()
deployment.flavor_name = 'flavor_with_extra_specs'
deployment.openstack_client = nova_client_mock
accommodation = {'compute_nodes': 3}
expected = [{'host': 'host-2', 'zone': 'nova'}]
observed = deployment._get_compute_nodes(accommodation)
self.assertEqual(expected, observed)
@mock.patch('shaker.openstack.clients.nova.get_available_compute_nodes')
def test_get_compute_nodes_non_admin(self, nova_nodes_mock):
deployment = deploy.Deployment()
deployment.flavor_name = 'test.flavor'
deployment.openstack_client = mock.Mock()
def raise_error(arg):
def raise_error(nova_client, flavor_name):
raise nova.ForbiddenException('err')
nova_nodes_mock.side_effect = raise_error
@ -581,9 +696,10 @@ class TestDeploy(testtools.TestCase):
@mock.patch('shaker.openstack.clients.nova.get_available_compute_nodes')
def test_get_compute_nodes_non_admin_zones(self, nova_nodes_mock):
deployment = deploy.Deployment()
deployment.flavor_name = 'test.flavor'
deployment.openstack_client = mock.Mock()
def raise_error(arg):
def raise_error(nova_client, flavor_name):
raise nova.ForbiddenException('err')
nova_nodes_mock.side_effect = raise_error
@ -602,9 +718,10 @@ class TestDeploy(testtools.TestCase):
@mock.patch('shaker.openstack.clients.nova.get_available_compute_nodes')
def test_get_compute_nodes_non_admin_not_configured(self, nova_nodes_mock):
deployment = deploy.Deployment()
deployment.flavor_name = 'test.flavor'
deployment.openstack_client = mock.Mock()
def raise_error(arg):
def raise_error(nova_client, flavor_name):
raise nova.ForbiddenException('err')
nova_nodes_mock.side_effect = raise_error