openstack
/
zun
Code Issues Proposed changes

Merge "Add scheduler hint"

This commit is contained in:
Jenkins 2017-06-08 16:08:25 +00:00 committed by Gerrit Code Review
parent f19eb3507c 478e72dd74
commit fa353a4c8a
20 changed files with 76 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
zun/api/controllers/v1/containers.py
View File

@ -252,13 +252,14 @@ class ContainersController(base.Controller):
if container_dict.get('restart_policy'):
self._check_for_restart_policy(container_dict)
container_dict['status'] = consts.CREATING
extra_spec = container_dict.get('hints', None)
new_container = objects.Container(context, **container_dict)
new_container.create(context)
if run:
compute_api.container_run(context, new_container)
compute_api.container_run(context, new_container, extra_spec)
else:
compute_api.container_create(context, new_container)
compute_api.container_create(context, new_container, extra_spec)
# Set the HTTP Location Header
pecan.response.location = link.build_url('containers',
new_container.uuid)

3
zun/api/controllers/v1/schemas/containers.py
View File

@ -27,7 +27,8 @@ _container_properties = {
'restart_policy': parameter_types.restart_policy,
'interactive': parameter_types.boolean,
'image_driver': parameter_types.image_driver,
'security_groups': parameter_types.security_groups
'security_groups': parameter_types.security_groups,
'hints': parameter_types.hints
}
container_create = {

4
zun/common/validation/parameter_types.py
View File

@ -101,6 +101,10 @@ labels = {
'type': ['object', 'null']
}
hints = {
'type': ['object', 'null']
}
environment = {
'type': ['object', 'null']
}

13
zun/compute/api.py
View File

@ -28,9 +28,9 @@ class API(object):
self.scheduler_client = scheduler_client.SchedulerClient()
super(API, self).__init__()
def container_create(self, context, new_container):
def container_create(self, context, new_container, extra_spec):
try:
self._schedule_container(context, new_container)
self._schedule_container(context, new_container, extra_spec)
except Exception as exc:
new_container.status = consts.ERROR
new_container.status_reason = str(exc)
@ -39,9 +39,9 @@ class API(object):
self.rpcapi.container_create(context, new_container)
def container_run(self, context, new_container):
def container_run(self, context, new_container, extra_spec):
try:
self._schedule_container(context, new_container)
self._schedule_container(context, new_container, extra_spec)
except Exception as exc:
new_container.status = consts.ERROR
new_container.status_reason = str(exc)
@ -50,9 +50,10 @@ class API(object):
self.rpcapi.container_run(context, new_container)
def _schedule_container(self, context, new_container):
def _schedule_container(self, context, new_container, extra_spec):
dests = self.scheduler_client.select_destinations(context,
[new_container])
[new_container],
extra_spec)
new_container.host = dests[0]['host']
new_container.save(context)

11
zun/scheduler/base_filters.py
View File

@ -23,11 +23,11 @@ LOG = logging.getLogger(__name__)
class BaseFilter(object):
"""Base class for all filter classes."""
def _filter_one(self, obj, container):
def _filter_one(self, obj, container, extra_spec):
"""Return True if it passes the filter, False otherwise."""
return True
def filter_all(self, filter_obj_list, container):
def filter_all(self, filter_obj_list, container, extra_spec):
"""Yield objects that pass the filter.
Can be overridden in a subclass, if you need to base filtering
@ -35,7 +35,7 @@ class BaseFilter(object):
_filter_one() to filter a single object.
"""
for obj in filter_obj_list:
if self._filter_one(obj, container):
if self._filter_one(obj, container, extra_spec):
yield obj
# Set to true in a subclass if a filter only needs to be run once
@ -61,7 +61,8 @@ class BaseFilterHandler(loadables.BaseLoader):
This class should be subclassed where one needs to use filters.
"""
def get_filtered_objects(self, filters, objs, container, index=0):
def get_filtered_objects(self, filters, objs, container, extra_spec,
index=0):
list_objs = list(objs)
LOG.debug("Starting with %d host(s)", len(list_objs))
part_filter_results = []
@ -71,7 +72,7 @@ class BaseFilterHandler(loadables.BaseLoader):
if filter_.run_filter_for_index(index):
cls_name = filter_.__class__.__name__
start_count = len(list_objs)
objs = filter_.filter_all(list_objs, container)
objs = filter_.filter_all(list_objs, container, extra_spec)
if objs is None:
LOG.debug("Filter %s says to stop filtering", cls_name)
return

2
zun/scheduler/chance_scheduler.py
View File

@ -33,7 +33,7 @@ class ChanceScheduler(driver.Scheduler):
return random.choice(hosts)
def select_destinations(self, context, containers):
def select_destinations(self, context, containers, extra_spec):
"""Selects random destinations."""
dests = []
for container in containers:

4
zun/scheduler/client.py
View File

@ -29,8 +29,8 @@ class SchedulerClient(object):
scheduler_driver,
invoke_on_load=True).driver
def select_destinations(self, context, containers):
return self.driver.select_destinations(context, containers)
def select_destinations(self, context, containers, extra_spec):
return self.driver.select_destinations(context, containers, extra_spec)
def update_resource(self, node):
node.save()

2
zun/scheduler/driver.py
View File

@ -47,6 +47,6 @@ class Scheduler(object):
"""Must override select_destinations method.
:return: A list of dicts with 'host', 'nodename' and 'limits' as keys
that satisfies the request_spec and filter_properties.
that satisfies the extra_spec and filter_properties.
"""
return []

9
zun/scheduler/filter_scheduler.py
View File

@ -40,25 +40,26 @@ class FilterScheduler(driver.Scheduler):
self.filter_obj_map = {}
self.enabled_filters = self._choose_host_filters(self._load_filters())
def _schedule(self, context, container):
def _schedule(self, context, container, extra_spec):
"""Picks a host according to filters."""
hosts = self.hosts_up(context)
nodes = objects.ComputeNode.list(context)
nodes = [node for node in nodes if node.hostname in hosts]
nodes = self.filter_handler.get_filtered_objects(self.enabled_filters,
nodes,
container)
container,
extra_spec)
if not nodes:
msg = _("Is the appropriate service running?")
raise exception.NoValidHost(reason=msg)
return random.choice(nodes)
def select_destinations(self, context, containers):
def select_destinations(self, context, containers, extra_spec):
"""Selects destinations by filters."""
dests = []
for container in containers:
node = self._schedule(context, container)
node = self._schedule(context, container, extra_spec)
host_state = dict(host=node.hostname, nodename=None, limits=None)
dests.append(host_state)

6
zun/scheduler/filters/__init__.py
View File

@ -18,11 +18,11 @@ from zun.scheduler import base_filters
class BaseHostFilter(base_filters.BaseFilter):
"""Base class for host filters."""
def _filter_one(self, obj, filter_properties):
def _filter_one(self, obj, filter_properties, extra_spec):
"""Return True if the object passes the filter, otherwise False."""
return self.host_passes(obj, filter_properties)
return self.host_passes(obj, filter_properties, extra_spec)
def host_passes(self, host_state, filter_properties):
def host_passes(self, host_state, filter_properties, extra_spec):
"""Return True if the HostState passes the filter,otherwise False.
Override this in a subclass.

2
zun/scheduler/filters/cpu_filter.py
View File

@ -25,7 +25,7 @@ class CPUFilter(filters.BaseHostFilter):
run_filter_once_per_request = True
def host_passes(self, host_state, container):
def host_passes(self, host_state, container, extra_spec):
cpu_free = host_state.cpus - host_state.cpu_used
if not container.cpu:
return True

2
zun/scheduler/filters/noop_filter.py
View File

@ -26,7 +26,7 @@ class NoopFilter(filters.BaseHostFilter):
# Host state does not change within a request
run_filter_once_per_request = True
def host_passes(self, host_state, container):
def host_passes(self, host_state, container, extra_spec):
"""Noop filter for now"""
# Depend on the objects.NodeInfo of below patch to filter node,

2
zun/scheduler/filters/ram_filter.py
View File

@ -25,7 +25,7 @@ class RamFilter(filters.BaseHostFilter):
run_filter_once_per_request = True
def host_passes(self, host_state, container):
def host_passes(self, host_state, container, extra_spec):
if not container.memory:
return True

32
zun/tests/unit/api/controllers/v1/test_containers.py
View File

@ -28,7 +28,7 @@ class TestContainerController(api_base.FunctionalTest):
@patch('zun.compute.api.API.container_run')
@patch('zun.compute.api.API.image_search')
def test_run_container(self, mock_search, mock_container_run):
mock_container_run.side_effect = lambda x, y: y
mock_container_run.side_effect = lambda x, y, z: y
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'
@ -56,7 +56,7 @@ class TestContainerController(api_base.FunctionalTest):
@patch('zun.compute.rpcapi.API.image_search')
def test_run_container_with_false(self, mock_search,
mock_container_run):
mock_container_run.side_effect = lambda x, y: y
mock_container_run.side_effect = lambda x, y, z: y
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'
@ -82,7 +82,7 @@ class TestContainerController(api_base.FunctionalTest):
@patch('zun.compute.api.API.container_create')
@patch('zun.compute.api.API.image_search')
def test_create_container(self, mock_search, mock_container_create):
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'
@ -111,7 +111,7 @@ class TestContainerController(api_base.FunctionalTest):
@patch('zun.compute.api.API.image_search')
def test_create_container_image_not_found(self, mock_search,
mock_container_create):
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
mock_search.side_effect = exception.ImageNotFound()
params = {"name": "MyDocker", "image": "not-found"}
@ -124,7 +124,7 @@ class TestContainerController(api_base.FunctionalTest):
@patch('zun.compute.api.API.image_search')
def test_create_container_set_project_id_and_user_id(
self, mock_search, mock_container_create):
def _create_side_effect(cnxt, container):
def _create_side_effect(cnxt, container, extra_spec):
self.assertEqual(self.context.project_id, container.project_id)
self.assertEqual(self.context.user_id, container.user_id)
return container
@ -141,7 +141,7 @@ class TestContainerController(api_base.FunctionalTest):
@patch('zun.compute.api.API.image_search')
def test_create_container_resp_has_status_reason(self, mock_search,
mock_container_create):
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
# Create a container with a command
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'
@ -160,7 +160,7 @@ class TestContainerController(api_base.FunctionalTest):
mock_container_delete,
mock_container_create,
mock_container_show):
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
# Create a container with a command
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'
@ -201,7 +201,7 @@ class TestContainerController(api_base.FunctionalTest):
def test_create_container_without_memory(self, mock_search,
mock_container_create,
mock_container_show):
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
# Create a container with a command
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env",'
@ -232,7 +232,7 @@ class TestContainerController(api_base.FunctionalTest):
def test_create_container_without_environment(self, mock_search,
mock_container_create,
mock_container_show):
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
# Create a container with a command
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512"}')
@ -262,7 +262,7 @@ class TestContainerController(api_base.FunctionalTest):
mock_container_create,
mock_container_show):
# No name param
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
params = ('{"image": "ubuntu", "command": "env", "memory": "512",'
'"environment": {"key1": "val1", "key2": "val2"}}')
response = self.app.post('/v1/containers/',
@ -293,7 +293,7 @@ class TestContainerController(api_base.FunctionalTest):
mock_search,
mock_container_create,
mock_container_show):
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
# Create a container with a command
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'
@ -327,7 +327,7 @@ class TestContainerController(api_base.FunctionalTest):
mock_search,
mock_container_create,
mock_container_show):
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
# Create a container with a command
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'
@ -361,7 +361,7 @@ class TestContainerController(api_base.FunctionalTest):
mock_search,
mock_container_create,
mock_container_show):
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
# Create a container with a command
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'
@ -394,7 +394,7 @@ class TestContainerController(api_base.FunctionalTest):
mock_search,
mock_container_create,
mock_container_show):
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
# Create a container with a command
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'
@ -428,7 +428,7 @@ class TestContainerController(api_base.FunctionalTest):
mock_search,
mock_container_create,
mock_container_show):
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
# Create a container with a command
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'
@ -1030,7 +1030,7 @@ class TestContainerController(api_base.FunctionalTest):
@patch('zun.compute.api.API.image_search')
def test_create_container_resp_has_image_driver(self, mock_search,
mock_container_create):
mock_container_create.side_effect = lambda x, y: y
mock_container_create.side_effect = lambda x, y, z: y
# Create a container with a command
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'

7
zun/tests/unit/scheduler/filters/test_cpu_filter.py
View File

@ -29,7 +29,8 @@ class TestCPUFilter(base.TestCase):
host = objects.ComputeNode(self.context)
host.cpus = 8
host.cpu_used = 0.0
self.assertTrue(self.filt_cls.host_passes(host, container))
extra_spec = {}
self.assertTrue(self.filt_cls.host_passes(host, container, extra_spec))
def test_cpu_filter_fail(self):
self.filt_cls = cpu_filter.CPUFilter()
@ -38,4 +39,6 @@ class TestCPUFilter(base.TestCase):
host = objects.ComputeNode(self.context)
host.cpus = 5
host.cpu_used = 2.0
self.assertFalse(self.filt_cls.host_passes(host, container))
extra_spec = {}
self.assertFalse(self.filt_cls.host_passes(host, container,
extra_spec))

7
zun/tests/unit/scheduler/filters/test_ram_filter.py
View File

@ -29,7 +29,8 @@ class TestRamFilter(base.TestCase):
host = objects.ComputeNode(self.context)
host.mem_total = 1024 * 128
host.mem_used = 1024
self.assertTrue(self.filt_cls.host_passes(host, container))
extra_spec = {}
self.assertTrue(self.filt_cls.host_passes(host, container, extra_spec))
def test_ram_filter_fail(self):
self.filt_cls = ram_filter.RamFilter()
@ -38,4 +39,6 @@ class TestRamFilter(base.TestCase):
host = objects.ComputeNode(self.context)
host.mem_total = 1024 * 128
host.mem_used = 1024 * 127
self.assertFalse(self.filt_cls.host_passes(host, container))
extra_spec = {}
self.assertFalse(self.filt_cls.host_passes(host, container,
extra_spec))

4
zun/tests/unit/scheduler/test_base_filter.py
View File

@ -29,8 +29,8 @@ class BaseFilterTestCase(base.TestCase):
filter_obj_list = ['obj1', 'obj2', 'obj3']
container = {}
base_filter = base_filters.BaseFilter()
result = base_filter.filter_all(filter_obj_list, container)
extra_spec = {}
result = base_filter.filter_all(filter_obj_list, container, extra_spec)
self.assertTrue(inspect.isgenerator(result))
self.assertEqual(['obj1', 'obj3'], list(result))

7
zun/tests/unit/scheduler/test_chance_scheduler.py
View File

@ -37,7 +37,9 @@ class ChanceSchedulerTestCase(base.TestCase):
test_container = utils.get_test_container()
containers = [objects.Container(self.context, **test_container)]
dests = self.driver_cls().select_destinations(self.context, containers)
extra_spec = {}
dests = self.driver_cls().select_destinations(self.context, containers,
extra_spec)
self.assertEqual(1, len(dests))
(host, node) = (dests[0]['host'], dests[0]['nodename'])
@ -56,6 +58,7 @@ class ChanceSchedulerTestCase(base.TestCase):
mock_hosts_up.side_effect = _return_no_host
test_container = utils.get_test_container()
containers = [objects.Container(self.context, **test_container)]
extra_spec = {}
self.assertRaises(exception.NoValidHost,
self.driver_cls().select_destinations, self.context,
containers)
containers, extra_spec)

2
zun/tests/unit/scheduler/test_client.py
View File

@ -42,6 +42,6 @@ class SchedulerClientTestCase(base.TestCase):
@mock.patch('zun.scheduler.filter_scheduler.FilterScheduler'
'.select_destinations')
def test_select_destinations(self, mock_select_destinations):
fake_args = ['ctxt', 'fake_containers']
fake_args = ['ctxt', 'fake_containers', 'fake_extra_spec']
self.client.select_destinations(*fake_args)
mock_select_destinations.assert_called_once_with(*fake_args)

7
zun/tests/unit/scheduler/test_filter_scheduler.py
View File

@ -82,7 +82,9 @@ class FilterSchedulerTestCase(base.TestCase):
nodes = [node1, node2, node3, node4]
mock_compute_list.return_value = nodes
mock_random_choice.side_effect = [node3]
dests = self.driver.select_destinations(self.context, containers)
extra_spec = {}
dests = self.driver.select_destinations(self.context, containers,
extra_spec)
self.assertEqual(1, len(dests))
(host, node) = (dests[0]['host'], dests[0]['nodename'])
@ -107,6 +109,7 @@ class FilterSchedulerTestCase(base.TestCase):
mock_list_by_binary.side_effect = _return_services
test_container = utils.get_test_container()
containers = [objects.Container(self.context, **test_container)]
extra_spec = {}
self.assertRaises(exception.NoValidHost,
self.driver.select_destinations, self.context,
containers)
containers, extra_spec)