Add tenant rack constraint to solver scheduler
This helps to limit the maximum number of racks that each tenant's VMs can spread across. The max rack number can be set via config option, and if the limit is not reached for a tenant the constraint logic will automatically look for additional rack(s) and mark hosts in the additional rack(s) as available. Change-Id: I792ac2e0ad75a6df8e8874b159a26760c47cfbd7
This commit is contained in:
Xinyuan Huang
parent
0ba1e05467
commit
fefcef5d67
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
import six
|
||||
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
|
||||
from nova.compute import api as compute
|
||||
|
|
@ -22,9 +23,76 @@ from nova.i18n import _
|
|||
from nova_solverscheduler.scheduler.solvers import constraints
|
||||
from nova_solverscheduler.scheduler.solvers import utils as solver_utils
|
||||
|
||||
rack_affinity_opts = [
|
||||
cfg.IntOpt('max_racks_per_tenant',
|
||||
default=1,
|
||||
help='Maximum number of racks each tenant can have.'),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(rack_affinity_opts)
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_sorted_racks(racks, hosts, host_racks_map, filter_properties):
|
||||
"""sort racks by total acceptable instance nums, then avg costs."""
|
||||
|
||||
def mixed_order(item):
|
||||
return (-item[1], item[2])
|
||||
|
||||
num_hosts = len(hosts)
|
||||
num_instances = filter_properties.get('num_instances')
|
||||
|
||||
solver_cache = filter_properties.get('solver_cache') or {}
|
||||
constraint_matrix = solver_cache.get('constraint_matrix', None)
|
||||
cost_matrix = solver_cache.get('cost_matrix', None)
|
||||
|
||||
if not constraint_matrix:
|
||||
return list(racks)
|
||||
if not cost_matrix:
|
||||
cost_matrix = [[0 for j in xrange(num_instances)]
|
||||
for i in xrange(num_hosts)]
|
||||
|
||||
rack_avail_insts = {}
|
||||
rack_avg_costs = {}
|
||||
rack_num_hosts = {}
|
||||
rack_set = set([])
|
||||
|
||||
for i in xrange(len(hosts)):
|
||||
host_name = hosts[i].host
|
||||
host_racks = host_racks_map.get(host_name, set())
|
||||
for rack in host_racks:
|
||||
if rack in racks:
|
||||
rack_set.add(rack)
|
||||
|
||||
# get maximum available instances number for each rack
|
||||
cons = constraint_matrix[i]
|
||||
host_max_avail_insts = 0
|
||||
for j in xrange(len(cons)):
|
||||
if cons[j] is True:
|
||||
host_max_avail_insts = j + 1
|
||||
rack_avail_insts.setdefault(rack, 0)
|
||||
rack_avail_insts[rack] += host_max_avail_insts
|
||||
|
||||
rack_num_hosts.setdefault(rack, 0)
|
||||
rack_num_hosts[rack] += 1
|
||||
|
||||
rack_avg_costs.setdefault(rack, 0)
|
||||
n = rack_num_hosts[rack]
|
||||
rack_avg_costs[rack] = (rack_avg_costs[rack] * (n - 1) +
|
||||
cost_matrix[i][0]) / n
|
||||
|
||||
rack_score_tuples = [
|
||||
(rack, rack_avail_insts[rack], rack_avg_costs[rack]) for
|
||||
rack in rack_set]
|
||||
|
||||
sorted_rack_tuples = sorted(rack_score_tuples, key=mixed_order)
|
||||
sorted_racks = [rack for (rack, inst, cost) in sorted_rack_tuples]
|
||||
|
||||
return sorted_racks
|
||||
|
||||
|
||||
class SameRackConstraint(constraints.BaseLinearConstraint):
|
||||
"""Place instances in the same racks as those of specified instances.
|
||||
If the specified instances are in hosts without rack config, then place
|
||||
|
|
@ -128,3 +196,67 @@ class DifferentRackConstraint(constraints.BaseLinearConstraint):
|
|||
{'host': host_name})
|
||||
|
||||
return constraint_matrix
|
||||
|
||||
|
||||
class TenantRackConstraint(constraints.BaseLinearConstraint):
|
||||
"""Limit the maximum number of racks that instances of each tenant can
|
||||
spread across.
|
||||
If a host doesnot have rack config, it won't be filtered out by this
|
||||
constraint and will always be regarded as compatible with rack limit.
|
||||
"""
|
||||
|
||||
precedence = 1
|
||||
|
||||
def get_constraint_matrix(self, hosts, filter_properties):
|
||||
num_hosts = len(hosts)
|
||||
num_instances = filter_properties.get('num_instances')
|
||||
|
||||
constraint_matrix = [[True for j in xrange(num_instances)]
|
||||
for i in xrange(num_hosts)]
|
||||
|
||||
max_racks = CONF.max_racks_per_tenant
|
||||
project_id = filter_properties['project_id']
|
||||
|
||||
host_racks_map = solver_utils.get_host_racks_map(hosts)
|
||||
|
||||
project_hosts = set([])
|
||||
project_racks = set([])
|
||||
other_racks = set([])
|
||||
|
||||
for i in xrange(num_hosts):
|
||||
host_name = hosts[i].host
|
||||
host_racks = host_racks_map.get(host_name, set([]))
|
||||
if project_id in hosts[i].projects:
|
||||
project_racks = project_racks.union(host_racks)
|
||||
project_hosts.add(host_name)
|
||||
else:
|
||||
other_racks = other_racks.union(host_racks)
|
||||
other_racks = other_racks.difference(project_racks)
|
||||
|
||||
additional_racks = []
|
||||
if len(project_racks) < max_racks:
|
||||
additional_rack_num = max_racks - len(project_racks)
|
||||
if additional_rack_num >= len(other_racks):
|
||||
additional_racks = list(other_racks)
|
||||
else:
|
||||
sorted_other_racks = _get_sorted_racks(
|
||||
other_racks, hosts, host_racks_map, filter_properties)
|
||||
additional_racks = sorted_other_racks[0:additional_rack_num]
|
||||
|
||||
acceptable_racks = project_racks.union(additional_racks)
|
||||
for i in xrange(num_hosts):
|
||||
host_name = hosts[i].host
|
||||
host_racks = host_racks_map.get(host_name, set([]))
|
||||
if (any([rack not in acceptable_racks for rack in host_racks])
|
||||
and (host_name not in project_hosts)):
|
||||
constraint_matrix[i] = [False for j in xrange(num_instances)]
|
||||
|
||||
LOG.debug(_("%(host)s cannot accept requested instances "
|
||||
"according to TenantRackAffinityConstraint."),
|
||||
{'host': host_name})
|
||||
else:
|
||||
LOG.debug(_("%(host)s can accept requested instances "
|
||||
"according to TenantRackAffinityConstraint."),
|
||||
{'host': host_name})
|
||||
|
||||
return constraint_matrix
|
||||
|
|
|
|||
|
|
@ -24,6 +24,197 @@ from nova_solverscheduler.tests.scheduler import solver_scheduler_fakes \
|
|||
as fakes
|
||||
|
||||
|
||||
class TestRackSorting(test.NoDBTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestRackSorting, self).setUp()
|
||||
self.fake_hosts = [fakes.FakeSolverSchedulerHostState(
|
||||
'host%s' % i, 'node1', {}) for i in xrange(1, 7)]
|
||||
|
||||
def test_get_sorted_racks_normal(self):
|
||||
"""Let number of available instances: rack1 = rack3 < rack2, and
|
||||
present host costs: rack3 < rack2 < rack1. The sorted racks should
|
||||
be: rack2, rack3, rack1.
|
||||
"""
|
||||
fake_racks_list = ['rack1', 'rack2', 'rack3']
|
||||
|
||||
fake_cost_matrix = [
|
||||
[5, 5, 5],
|
||||
[4, 4, 4],
|
||||
[3, 3, 3],
|
||||
[2, 2, 2],
|
||||
[1, 1, 1],
|
||||
[0, 0, 0]
|
||||
]
|
||||
fake_constraint_matrix = [
|
||||
[True, True, False],
|
||||
[True, False, False],
|
||||
[True, True, True],
|
||||
[True, True, True],
|
||||
[False, False, False],
|
||||
[True, True, True]
|
||||
]
|
||||
|
||||
fake_filter_properties = {
|
||||
'num_instances': 3,
|
||||
'solver_cache': {'cost_matrix': fake_cost_matrix,
|
||||
'constraint_matrix': fake_constraint_matrix}
|
||||
}
|
||||
|
||||
fake_host_racks_map = {
|
||||
'host1': set(['rack1']),
|
||||
'host2': set(['rack1']),
|
||||
'host3': set(['rack2']),
|
||||
'host4': set(['rack2']),
|
||||
'host5': set(['rack3']),
|
||||
'host6': set(['rack3'])
|
||||
}
|
||||
|
||||
expected_result = ['rack2', 'rack3', 'rack1']
|
||||
result = rack_affinity_constraint._get_sorted_racks(fake_racks_list,
|
||||
self.fake_hosts, fake_host_racks_map, fake_filter_properties)
|
||||
self.assertEqual(expected_result, result)
|
||||
|
||||
def test_get_sorted_racks_normal_shorter_input_racks_list(self):
|
||||
fake_racks_list = ['rack1', 'rack2']
|
||||
|
||||
fake_cost_matrix = [
|
||||
[5, 5, 5],
|
||||
[4, 4, 4],
|
||||
[3, 3, 3],
|
||||
[2, 2, 2],
|
||||
[1, 1, 1],
|
||||
[0, 0, 0]
|
||||
]
|
||||
fake_constraint_matrix = [
|
||||
[True, True, False],
|
||||
[True, False, False],
|
||||
[True, True, True],
|
||||
[True, True, True],
|
||||
[False, False, False],
|
||||
[True, True, True]
|
||||
]
|
||||
|
||||
fake_filter_properties = {
|
||||
'num_instances': 3,
|
||||
'solver_cache': {'cost_matrix': fake_cost_matrix,
|
||||
'constraint_matrix': fake_constraint_matrix}
|
||||
}
|
||||
|
||||
fake_host_racks_map = {
|
||||
'host1': set(['rack1']),
|
||||
'host2': set(['rack1']),
|
||||
'host3': set(['rack2']),
|
||||
'host4': set(['rack2']),
|
||||
'host5': set(['rack3']),
|
||||
'host6': set(['rack3'])
|
||||
}
|
||||
|
||||
expected_result = ['rack2', 'rack1']
|
||||
result = rack_affinity_constraint._get_sorted_racks(fake_racks_list,
|
||||
self.fake_hosts, fake_host_racks_map, fake_filter_properties)
|
||||
self.assertEqual(expected_result, result)
|
||||
|
||||
def test_get_sorted_racks_no_cost_matrix(self):
|
||||
fake_racks_list = ['rack1', 'rack2', 'rack3']
|
||||
|
||||
fake_cost_matrix = []
|
||||
fake_constraint_matrix = [
|
||||
[True, True, False],
|
||||
[True, False, False],
|
||||
[True, True, True],
|
||||
[True, True, True],
|
||||
[False, False, False],
|
||||
[True, True, True]
|
||||
]
|
||||
|
||||
fake_filter_properties = {
|
||||
'num_instances': 3,
|
||||
'solver_cache': {'cost_matrix': fake_cost_matrix,
|
||||
'constraint_matrix': fake_constraint_matrix}
|
||||
}
|
||||
|
||||
fake_host_racks_map = {
|
||||
'host1': set(['rack1']),
|
||||
'host2': set(['rack1']),
|
||||
'host3': set(['rack2']),
|
||||
'host4': set(['rack2']),
|
||||
'host5': set(['rack3']),
|
||||
'host6': set(['rack3'])
|
||||
}
|
||||
|
||||
expected_result_idx0 = 'rack2'
|
||||
result = rack_affinity_constraint._get_sorted_racks(fake_racks_list,
|
||||
self.fake_hosts, fake_host_racks_map, fake_filter_properties)
|
||||
self.assertEqual(expected_result_idx0, result[0])
|
||||
|
||||
def test_get_sorted_racks_no_constraint_matrix(self):
|
||||
fake_racks_list = ['rack1', 'rack2', 'rack3']
|
||||
|
||||
fake_cost_matrix = [
|
||||
[5, 5, 5],
|
||||
[4, 4, 4],
|
||||
[3, 3, 3],
|
||||
[2, 2, 2],
|
||||
[1, 1, 1],
|
||||
[0, 0, 0]
|
||||
]
|
||||
fake_constraint_matrix = []
|
||||
|
||||
fake_filter_properties = {
|
||||
'num_instances': 3,
|
||||
'solver_cache': {'cost_matrix': fake_cost_matrix,
|
||||
'constraint_matrix': fake_constraint_matrix}
|
||||
}
|
||||
|
||||
fake_host_racks_map = {
|
||||
'host1': set(['rack1']),
|
||||
'host2': set(['rack1']),
|
||||
'host3': set(['rack2']),
|
||||
'host4': set(['rack2']),
|
||||
'host5': set(['rack3']),
|
||||
'host6': set(['rack3'])
|
||||
}
|
||||
|
||||
expected_result = ['rack1', 'rack2', 'rack3']
|
||||
result = rack_affinity_constraint._get_sorted_racks(fake_racks_list,
|
||||
self.fake_hosts, fake_host_racks_map, fake_filter_properties)
|
||||
self.assertEqual(expected_result, result)
|
||||
|
||||
def test_get_sorted_racks_no_host_racks_map(self):
|
||||
fake_racks_list = ['rack1', 'rack2', 'rack3']
|
||||
|
||||
fake_cost_matrix = [
|
||||
[5, 5, 5],
|
||||
[4, 4, 4],
|
||||
[3, 3, 3],
|
||||
[2, 2, 2],
|
||||
[1, 1, 1],
|
||||
[0, 0, 0]
|
||||
]
|
||||
fake_constraint_matrix = [
|
||||
[True, True, False],
|
||||
[True, False, False],
|
||||
[True, True, True],
|
||||
[True, True, True],
|
||||
[False, False, False],
|
||||
[True, True, True]
|
||||
]
|
||||
|
||||
fake_filter_properties = {
|
||||
'num_instances': 3,
|
||||
'solver_cache': {'cost_matrix': fake_cost_matrix,
|
||||
'constraint_matrix': fake_constraint_matrix}
|
||||
}
|
||||
|
||||
fake_host_racks_map = {}
|
||||
|
||||
expected_result = []
|
||||
result = rack_affinity_constraint._get_sorted_racks(fake_racks_list,
|
||||
self.fake_hosts, fake_host_racks_map, fake_filter_properties)
|
||||
self.assertEqual(expected_result, result)
|
||||
|
||||
|
||||
@mock.patch('nova_solverscheduler.scheduler.solvers.utils.get_host_racks_map')
|
||||
class TestSameRackConstraint(test.NoDBTestCase):
|
||||
USES_DB = True
|
||||
|
|
@ -463,3 +654,225 @@ class TestDifferentRackConstraint(test.NoDBTestCase):
|
|||
self.fake_hosts, fake_filter_properties)
|
||||
|
||||
self.assertEqual(expected_cons_mat, cons_mat)
|
||||
|
||||
|
||||
@mock.patch('nova_solverscheduler.scheduler.solvers.utils.get_host_racks_map')
|
||||
class TestTenantRackConstraint(test.NoDBTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestTenantRackConstraint, self).setUp()
|
||||
self.constraint_cls = rack_affinity_constraint.\
|
||||
TenantRackConstraint
|
||||
self.context = context.RequestContext('fake', 'fake')
|
||||
self.fake_hosts = [fakes.FakeSolverSchedulerHostState(
|
||||
'host%s' % i, 'node1', {'projects': []})
|
||||
for i in xrange(1, 7)]
|
||||
|
||||
def test_tenant_rack_max_racks_reached(self, racks_mock):
|
||||
self.flags(max_racks_per_tenant=2)
|
||||
|
||||
# let the tenant vm's be in host1, host3
|
||||
self.fake_hosts[0].projects = ['fake']
|
||||
self.fake_hosts[2].projects = ['fake']
|
||||
|
||||
fake_filter_properties = {
|
||||
'context': self.context,
|
||||
'project_id': 'fake',
|
||||
'num_instances': 2
|
||||
}
|
||||
|
||||
racks_mock.return_value = {
|
||||
'host1': set(['rack1']),
|
||||
'host2': set(['rack1']),
|
||||
'host3': set(['rack2']),
|
||||
'host4': set(['rack2']),
|
||||
'host5': set(['rack3']),
|
||||
'host6': set(['rack3'])
|
||||
}
|
||||
|
||||
expected_cons_mat = [
|
||||
[True, True],
|
||||
[True, True],
|
||||
[True, True],
|
||||
[True, True],
|
||||
[False, False],
|
||||
[False, False]
|
||||
]
|
||||
|
||||
cons_mat = self.constraint_cls().get_constraint_matrix(
|
||||
self.fake_hosts, fake_filter_properties)
|
||||
|
||||
self.assertEqual(expected_cons_mat, cons_mat)
|
||||
|
||||
def test_tenant_rack_choose_additional_racks1(self, racks_mock):
|
||||
"""sort additional racks by available instances number"""
|
||||
self.flags(max_racks_per_tenant=2)
|
||||
|
||||
# let the tenant vm's be in host1
|
||||
self.fake_hosts[0].projects = ['fake']
|
||||
|
||||
fake_cost_matrix = [
|
||||
[0, 1],
|
||||
[1, 2],
|
||||
[2, 3],
|
||||
[3, 4],
|
||||
[4, 5],
|
||||
[5, 6]
|
||||
]
|
||||
fake_constraint_matrix = [
|
||||
[True, True],
|
||||
[True, True],
|
||||
[True, False],
|
||||
[False, False],
|
||||
[True, True],
|
||||
[False, False]
|
||||
]
|
||||
|
||||
fake_filter_properties = {
|
||||
'context': self.context,
|
||||
'project_id': 'fake',
|
||||
'num_instances': 2,
|
||||
'solver_cache': {'cost_matrix': fake_cost_matrix,
|
||||
'constraint_matrix': fake_constraint_matrix}
|
||||
}
|
||||
|
||||
racks_mock.return_value = {
|
||||
'host1': set(['rack1']),
|
||||
'host2': set(['rack1']),
|
||||
'host3': set(['rack2']),
|
||||
'host4': set(['rack2']),
|
||||
'host5': set(['rack3']),
|
||||
'host6': set(['rack3'])
|
||||
}
|
||||
|
||||
expected_cons_mat = [
|
||||
[True, True],
|
||||
[True, True],
|
||||
[False, False],
|
||||
[False, False],
|
||||
[True, True],
|
||||
[True, True]
|
||||
]
|
||||
|
||||
cons_mat = self.constraint_cls().get_constraint_matrix(
|
||||
self.fake_hosts, fake_filter_properties)
|
||||
|
||||
self.assertEqual(expected_cons_mat, cons_mat)
|
||||
|
||||
def test_tenant_rack_choose_additional_racks2(self, racks_mock):
|
||||
"""sort additional racks by costs when available num_hosts are same."""
|
||||
self.flags(max_racks_per_tenant=2)
|
||||
|
||||
# let the tenant vm's be in host1
|
||||
self.fake_hosts[0].projects = ['fake']
|
||||
|
||||
fake_cost_matrix = [
|
||||
[0, 1],
|
||||
[1, 2],
|
||||
[2, 3],
|
||||
[3, 4],
|
||||
[4, 5],
|
||||
[5, 6]
|
||||
]
|
||||
fake_constraint_matrix = [
|
||||
[True, True],
|
||||
[True, True],
|
||||
[True, True],
|
||||
[False, False],
|
||||
[True, True],
|
||||
[False, False]
|
||||
]
|
||||
|
||||
fake_filter_properties = {
|
||||
'context': self.context,
|
||||
'project_id': 'fake',
|
||||
'num_instances': 2,
|
||||
'solver_cache': {'cost_matrix': fake_cost_matrix,
|
||||
'constraint_matrix': fake_constraint_matrix}
|
||||
}
|
||||
|
||||
racks_mock.return_value = {
|
||||
'host1': set(['rack1']),
|
||||
'host2': set(['rack1']),
|
||||
'host3': set(['rack2']),
|
||||
'host4': set(['rack2']),
|
||||
'host5': set(['rack3']),
|
||||
'host6': set(['rack3'])
|
||||
}
|
||||
|
||||
expected_cons_mat = [
|
||||
[True, True],
|
||||
[True, True],
|
||||
[True, True],
|
||||
[True, True],
|
||||
[False, False],
|
||||
[False, False]
|
||||
]
|
||||
|
||||
cons_mat = self.constraint_cls().get_constraint_matrix(
|
||||
self.fake_hosts, fake_filter_properties)
|
||||
|
||||
self.assertEqual(expected_cons_mat, cons_mat)
|
||||
|
||||
def test_tenant_rack_incomplete_rack_config(self, racks_mock):
|
||||
self.flags(max_racks_per_tenant=1)
|
||||
|
||||
# let the tenant vm's be in host2, host3
|
||||
self.fake_hosts[1].projects = ['fake']
|
||||
self.fake_hosts[2].projects = ['fake']
|
||||
|
||||
fake_filter_properties = {
|
||||
'context': self.context,
|
||||
'project_id': 'fake',
|
||||
'num_instances': 2
|
||||
}
|
||||
|
||||
racks_mock.return_value = {
|
||||
'host1': set(['rack1']),
|
||||
'host3': set(['rack2']),
|
||||
'host4': set(['rack2']),
|
||||
'host6': set(['rack3'])
|
||||
}
|
||||
|
||||
expected_cons_mat = [
|
||||
[False, False],
|
||||
[True, True],
|
||||
[True, True],
|
||||
[True, True],
|
||||
[True, True],
|
||||
[False, False]
|
||||
]
|
||||
|
||||
cons_mat = self.constraint_cls().get_constraint_matrix(
|
||||
self.fake_hosts, fake_filter_properties)
|
||||
|
||||
self.assertEqual(expected_cons_mat, cons_mat)
|
||||
|
||||
def test_tenant_rack_no_rack_config(self, racks_mock):
|
||||
self.flags(max_racks_per_tenant=1)
|
||||
|
||||
# let the tenant vm's be in host2, host3
|
||||
self.fake_hosts[1].projects = ['fake']
|
||||
self.fake_hosts[2].projects = ['fake']
|
||||
|
||||
fake_filter_properties = {
|
||||
'context': self.context,
|
||||
'project_id': 'fake',
|
||||
'num_instances': 2
|
||||
}
|
||||
|
||||
racks_mock.return_value = {}
|
||||
|
||||
expected_cons_mat = [
|
||||
[True, True],
|
||||
[True, True],
|
||||
[True, True],
|
||||
[True, True],
|
||||
[True, True],
|
||||
[True, True]
|
||||
]
|
||||
|
||||
cons_mat = self.constraint_cls().get_constraint_matrix(
|
||||
self.fake_hosts, fake_filter_properties)
|
||||
|
||||
self.assertEqual(expected_cons_mat, cons_mat)
|
||||
|
|
|
|||
Loading…
Reference in New Issue