From 9de2e37188e940138444cf763f8a81aa19929179 Mon Sep 17 00:00:00 2001 From: liusheng Date: Mon, 26 Jun 2017 11:10:55 +0800 Subject: [PATCH] Refactor the scheduler to use placement service This change change the scheduler to use placement api to filter query nodes with specified resource class of flavor, and clean the filters and weighers in scheduler. Change-Id: I89ad443f553510da6daf289b83f3c30d9d546ace Partially Implements: bp track-resources-using-placement --- mogan/common/service.py | 10 +- mogan/engine/api.py | 1 - mogan/engine/base_manager.py | 2 - mogan/engine/flows/create_server.py | 16 +- mogan/engine/manager.py | 12 +- mogan/scheduler/base_filter.py | 134 -------------- mogan/scheduler/base_handler.py | 47 ----- mogan/scheduler/base_weight.py | 145 --------------- mogan/scheduler/client/__init__.py | 9 + mogan/scheduler/client/query.py | 33 ++++ mogan/scheduler/client/report.py | 172 +++++++---------- mogan/scheduler/driver.py | 5 - mogan/scheduler/filter_scheduler.py | 107 ++++------- mogan/scheduler/filters/__init__.py | 39 ---- .../filters/availability_zone_filter.py | 31 ---- .../scheduler/filters/capabilities_filter.py | 86 --------- mogan/scheduler/filters/extra_specs_ops.py | 77 -------- mogan/scheduler/filters/flavor_filter.py | 32 ---- mogan/scheduler/filters/json_filter.py | 149 --------------- mogan/scheduler/filters/ports_filter.py | 50 ----- mogan/scheduler/manager.py | 8 +- mogan/scheduler/node_manager.py | 146 --------------- mogan/scheduler/scheduler_options.py | 95 ---------- mogan/scheduler/weights/__init__.py | 45 ----- mogan/scheduler/weights/port.py | 41 ----- .../engine/flows/test_create_server_flow.py | 6 +- mogan/tests/unit/scheduler/__init__.py | 0 mogan/tests/unit/scheduler/fakes.py | 34 ---- .../tests/unit/scheduler/test_base_filter.py | 174 ------------------ .../tests/unit/scheduler/test_node_manager.py | 93 ---------- mogan/tests/unit/scheduler/test_rpcapi.py | 94 ---------- .../unit/scheduler/test_scheduler_options.py | 138 -------------- mogan/tests/unit/scheduler/test_weights.py | 54 ------ setup.cfg | 9 - 34 files changed, 165 insertions(+), 1929 deletions(-) delete mode 100644 mogan/scheduler/base_filter.py delete mode 100644 mogan/scheduler/base_handler.py delete mode 100644 mogan/scheduler/base_weight.py create mode 100644 mogan/scheduler/client/query.py delete mode 100644 mogan/scheduler/filters/__init__.py delete mode 100644 mogan/scheduler/filters/availability_zone_filter.py delete mode 100644 mogan/scheduler/filters/capabilities_filter.py delete mode 100644 mogan/scheduler/filters/extra_specs_ops.py delete mode 100644 mogan/scheduler/filters/flavor_filter.py delete mode 100644 mogan/scheduler/filters/json_filter.py delete mode 100644 mogan/scheduler/filters/ports_filter.py delete mode 100644 mogan/scheduler/node_manager.py delete mode 100644 mogan/scheduler/scheduler_options.py delete mode 100644 mogan/scheduler/weights/__init__.py delete mode 100644 mogan/scheduler/weights/port.py delete mode 100644 mogan/tests/unit/scheduler/__init__.py delete mode 100644 mogan/tests/unit/scheduler/fakes.py delete mode 100644 mogan/tests/unit/scheduler/test_base_filter.py delete mode 100644 mogan/tests/unit/scheduler/test_node_manager.py delete mode 100644 mogan/tests/unit/scheduler/test_rpcapi.py delete mode 100644 mogan/tests/unit/scheduler/test_scheduler_options.py delete mode 100644 mogan/tests/unit/scheduler/test_weights.py diff --git a/mogan/common/service.py b/mogan/common/service.py index 17c6c955..0d974951 100644 --- a/mogan/common/service.py +++ b/mogan/common/service.py @@ -15,6 +15,7 @@ from oslo_concurrency import processutils from oslo_context import context from oslo_log import log import oslo_messaging as messaging +from oslo_service import periodic_task from oslo_service import service from oslo_service import wsgi from oslo_utils import importutils @@ -53,10 +54,11 @@ class RPCService(service.Service): self.rpcserver.start() self.manager.init_host() - self.tg.add_dynamic_timer( - self.manager.periodic_tasks, - periodic_interval_max=CONF.periodic_interval, - context=admin_context) + if isinstance(self.manager, periodic_task.PeriodicTasks): + self.tg.add_dynamic_timer( + self.manager.periodic_tasks, + periodic_interval_max=CONF.periodic_interval, + context=admin_context) LOG.info('Created RPC server for service %(service)s on host ' '%(host)s.', diff --git a/mogan/engine/api.py b/mogan/engine/api.py index e4c16217..66170191 100644 --- a/mogan/engine/api.py +++ b/mogan/engine/api.py @@ -286,7 +286,6 @@ class API(object): servers = self._provision_servers(context, base_options, min_count, max_count) request_spec = { - 'server_id': servers[0].uuid, 'server_properties': { 'flavor_uuid': servers[0].flavor_uuid, 'networks': requested_networks, diff --git a/mogan/engine/base_manager.py b/mogan/engine/base_manager.py index 20eb3129..5c8ff3f3 100644 --- a/mogan/engine/base_manager.py +++ b/mogan/engine/base_manager.py @@ -24,7 +24,6 @@ from mogan.db import api as dbapi from mogan.engine.baremetal import driver from mogan.engine import rpcapi from mogan import network -from mogan.scheduler import rpcapi as scheduler_rpcapi class BaseEngineManager(periodic_task.PeriodicTasks): @@ -36,7 +35,6 @@ class BaseEngineManager(periodic_task.PeriodicTasks): self.host = host self.topic = topic self.network_api = network.API() - self.scheduler_rpcapi = scheduler_rpcapi.SchedulerAPI() self.driver = driver.load_engine_driver(CONF.engine.engine_driver) self.engine_rpcapi = rpcapi.EngineAPI() self._sync_power_pool = greenpool.GreenPool( diff --git a/mogan/engine/flows/create_server.py b/mogan/engine/flows/create_server.py index 939be732..81a7c2c1 100644 --- a/mogan/engine/flows/create_server.py +++ b/mogan/engine/flows/create_server.py @@ -34,7 +34,7 @@ from mogan.common import utils from mogan.engine import configdrive from mogan.engine import metadata as server_metadata from mogan import objects - +from mogan.scheduler import client as sched_client LOG = logging.getLogger(__name__) @@ -64,6 +64,7 @@ class OnFailureRescheduleTask(flow_utils.MoganTask): exception.ServerDeployAborted, exception.NetworkError, ] + self.reportclient = sched_client.SchedulerClient().reportclient def execute(self, **kwargs): pass @@ -137,15 +138,16 @@ class BuildNetworkTask(flow_utils.MoganTask): """Build network for the server.""" def __init__(self, manager): - requires = ['server', 'requested_networks', 'ports', 'context'] + requires = ['server', 'requested_networks', 'context'] super(BuildNetworkTask, self).__init__(addons=[ACTION], requires=requires) self.manager = manager - def _build_networks(self, context, server, requested_networks, ports): + def _build_networks(self, context, server, requested_networks): # TODO(zhenguo): This seems not needed as our scheduler has already # guaranteed this. + ports = self.manager.driver.get_ports_from_node(server.node_uuid) if len(requested_networks) > len(ports): raise exception.InterfacePlugException(_( "Ironic node: %(id)s virtual to physical interface count" @@ -196,12 +198,11 @@ class BuildNetworkTask(flow_utils.MoganTask): return nics_obj - def execute(self, context, server, requested_networks, ports): + def execute(self, context, server, requested_networks): server_nics = self._build_networks( context, server, - requested_networks, - ports) + requested_networks) server.nics = server_nics server.save() @@ -298,7 +299,7 @@ class CreateServerTask(flow_utils.MoganTask): def get_flow(context, manager, server, requested_networks, user_data, - injected_files, key_pair, ports, request_spec, + injected_files, key_pair, request_spec, filter_properties): """Constructs and returns the manager entrypoint flow @@ -326,7 +327,6 @@ def get_flow(context, manager, server, requested_networks, user_data, 'user_data': user_data, 'injected_files': injected_files, 'key_pair': key_pair, - 'ports': ports, 'configdrive': {} } diff --git a/mogan/engine/manager.py b/mogan/engine/manager.py index 1e8abbb5..f93237a3 100644 --- a/mogan/engine/manager.py +++ b/mogan/engine/manager.py @@ -195,6 +195,9 @@ class EngineManager(base_manager.BaseEngineManager): self.scheduler_client.set_inventory_for_provider( node.uuid, node.name, inventory_data, resource_class) + if node.provision_state == 'available': + self.scheduler_client.reportclient \ + .delete_allocations_for_resource_provider(node.uuid) @periodic_task.periodic_task(spacing=CONF.engine.sync_power_state_interval, run_immediately=True) @@ -392,9 +395,10 @@ class EngineManager(base_manager.BaseEngineManager): } filter_properties['retry'] = retry request_spec['num_servers'] = len(servers) + request_spec['server_ids'] = [s.uuid for s in servers] try: - nodes = self.scheduler_rpcapi.select_destinations( + nodes = self.scheduler_client.select_destinations( context, request_spec, filter_properties) except exception.NoValidNode as e: # Here should reset the state of building servers to Error @@ -413,11 +417,11 @@ class EngineManager(base_manager.BaseEngineManager): {"nodes": nodes}) for (server, node) in six.moves.zip(servers, nodes): - server.node_uuid = node['node_uuid'] + server.node_uuid = node server.save() # Add a retry entry for the selected node retry_nodes = retry['nodes'] - retry_nodes.append(node['node_uuid']) + retry_nodes.append(node) for server in servers: utils.spawn_n(self._create_server, @@ -444,7 +448,6 @@ class EngineManager(base_manager.BaseEngineManager): target_state=states.ACTIVE) try: - node = objects.ComputeNode.get(context, server.node_uuid) flow_engine = create_server.get_flow( context, self, @@ -453,7 +456,6 @@ class EngineManager(base_manager.BaseEngineManager): user_data, injected_files, key_pair, - node['ports'], request_spec, filter_properties, ) diff --git a/mogan/scheduler/base_filter.py b/mogan/scheduler/base_filter.py deleted file mode 100644 index 952f5e2e..00000000 --- a/mogan/scheduler/base_filter.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright (c) 2011-2012 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Filter support -""" -from oslo_log import log as logging -import six - -from mogan.scheduler import base_handler - -LOG = logging.getLogger(__name__) - - -class BaseFilter(object): - """Base class for all filter classes.""" - def _filter_one(self, obj, filter_properties): - """Return True if it passes the filter, False otherwise. - - Override this in a subclass. - """ - return True - - def filter_all(self, filter_obj_list, filter_properties): - """Yield objects that pass the filter. - - Can be overridden in a subclass, if you need to base filtering - decisions on all objects. Otherwise, one can just override - _filter_one() to filter a single object. - """ - for obj in filter_obj_list: - if self._filter_one(obj, filter_properties): - yield obj - - # Set to true in a subclass if a filter only needs to be run once - # for each request rather than for each server - run_filter_once_per_request = False - - def run_filter_for_index(self, index): - """Return True if the filter needs to be run for n-th servers. - - Only need to override this if a filter needs anything other than - "first only" or "all" behaviour. - """ - return not (self.run_filter_once_per_request and index > 0) - - -class BaseFilterHandler(base_handler.BaseHandler): - """Base class to handle loading filter classes. - - This class should be subclassed where one needs to use filters. - """ - - def _log_filtration(self, full_filter_results, - part_filter_results, filter_properties): - # Log the filtration history - rspec = filter_properties.get("request_spec", {}) - msg_dict = {"server_id": rspec.get("server_id", ""), - "str_results": six.text_type(full_filter_results), - } - full_msg = ("Filtering removed all nodes for the request with " - "server ID " - "'%(server_id)s'. Filter results: %(str_results)s" - ) % msg_dict - msg_dict["str_results"] = ', '.join( - ("%(cls_name)s: (start: %(start)s, end: %(end)s)") % - {"cls_name": value[0], "start": value[1], "end": value[2]} - for value in part_filter_results) - part_msg = ("Filtering removed all nodes for the request with " - "server ID '%(server_id)s'. " - "Filter results: %(str_results)s") % msg_dict - LOG.debug(full_msg) - LOG.info(part_msg) - - def get_filtered_objects(self, filter_classes, objs, - filter_properties, index=0): - """Get objects after filter - - :param filter_classes: filters that will be used to filter the - objects - :param objs: objects that will be filtered - :param filter_properties: client filter properties - :param index: This value needs to be increased in the caller - function of get_filtered_objects when handling - each resource. - """ - list_objs = list(objs) - LOG.debug("Starting with %d node(s)", len(list_objs)) - # The 'part_filter_results' list just tracks the number of hosts - # before and after the filter, unless the filter returns zero - # hosts, in which it records the host/nodename for the last batch - # that was removed. Since the full_filter_results can be very large, - # it is only recorded if the LOG level is set to debug. - part_filter_results = [] - full_filter_results = [] - for filter_cls in filter_classes: - cls_name = filter_cls.__name__ - start_count = len(list_objs) - filter_class = filter_cls() - - if filter_class.run_filter_for_index(index): - objs = filter_class.filter_all(list_objs, filter_properties) - if objs is None: - LOG.info("Filter %s returned 0 nodes", cls_name) - full_filter_results.append((cls_name, None)) - list_objs = None - break - - list_objs = list(objs) - end_count = len(list_objs) - part_filter_results.append((cls_name, start_count, end_count)) - remaining = [getattr(obj, "node", obj) - for obj in list_objs] - full_filter_results.append((cls_name, remaining)) - - LOG.debug("Filter %(cls_name)s returned " - "%(obj_len)d node(s)", - {'cls_name': cls_name, 'obj_len': len(list_objs)}) - if not list_objs: - self._log_filtration(full_filter_results, - part_filter_results, filter_properties) - return list_objs diff --git a/mogan/scheduler/base_handler.py b/mogan/scheduler/base_handler.py deleted file mode 100644 index bc8e1421..00000000 --- a/mogan/scheduler/base_handler.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2011-2013 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -A common base for handling extension classes. - -Used by BaseFilterHandler and BaseWeightHandler -""" - -import inspect - -from stevedore import extension - - -class BaseHandler(object): - """Base class to handle loading filter and weight classes.""" - def __init__(self, modifier_class_type, modifier_namespace): - self.namespace = modifier_namespace - self.modifier_class_type = modifier_class_type - self.extension_manager = extension.ExtensionManager(modifier_namespace) - - def _is_correct_class(self, cls): - """Return whether an object is a class of the correct type. - - (or is not prefixed with an underscore) - """ - return (inspect.isclass(cls) and - not cls.__name__.startswith('_') and - issubclass(cls, self.modifier_class_type)) - - def get_all_classes(self): - # We use a set, as some classes may have an entrypoint of their own, - # and also be returned by a function such as 'all_filters' for example - return [ext.plugin for ext in self.extension_manager if - self._is_correct_class(ext.plugin)] diff --git a/mogan/scheduler/base_weight.py b/mogan/scheduler/base_weight.py deleted file mode 100644 index e6badcf2..00000000 --- a/mogan/scheduler/base_weight.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright (c) 2011-2012 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Pluggable Weighing support -""" - -import abc - -import six - -from mogan.scheduler import base_handler - - -def normalize(weight_list, minval=None, maxval=None): - """Normalize the values in a list between 0 and 1.0. - - The normalization is made regarding the lower and upper values present in - weight_list. If the minval and/or maxval parameters are set, these values - will be used instead of the minimum and maximum from the list. - - If all the values are equal, they are normalized to 0. - """ - - if not weight_list: - return () - - if maxval is None: - maxval = max(weight_list) - - if minval is None: - minval = min(weight_list) - - maxval = float(maxval) - minval = float(minval) - - if minval == maxval: - return [0] * len(weight_list) - - range_ = maxval - minval - return ((i - minval) / range_ for i in weight_list) - - -class WeighedObject(object): - """Object with weight information.""" - def __init__(self, obj, weight): - self.obj = obj - self.weight = weight - - def __repr__(self): - return "" % (self.obj, self.weight) - - -@six.add_metaclass(abc.ABCMeta) -class BaseWeigher(object): - """Base class for pluggable weighers. - - The attributes maxval and minval can be specified to set up the maximum - and minimum values for the weighed objects. These values will then be - taken into account in the normalization step, instead of taking the values - from the calculated weights. - """ - - minval = None - maxval = None - - def weight_multiplier(self): - """How weighted this weigher should be. - - Override this method in a subclass, so that the returned value is - read from a configuration option to permit operators specify a - multiplier for the weigher. - """ - return 1.0 - - @abc.abstractmethod - def _weigh_object(self, obj, weight_properties): - """Override in a subclass to specify a weight for a specific object.""" - - def weigh_objects(self, weighed_obj_list, weight_properties): - """Weigh multiple objects. - - Override in a subclass if you need access to all objects in order - to calculate weights. Do not modify the weight of an object here, - just return a list of weights. - """ - # Calculate the weights - weights = [] - for obj in weighed_obj_list: - weight = self._weigh_object(obj.obj, weight_properties) - - # Record the min and max values if they are None. If they anything - # but none we assume that the weigher has set them - if self.minval is None: - self.minval = weight - if self.maxval is None: - self.maxval = weight - - if weight < self.minval: - self.minval = weight - elif weight > self.maxval: - self.maxval = weight - - weights.append(weight) - - return weights - - -class BaseWeightHandler(base_handler.BaseHandler): - object_class = WeighedObject - - def get_weighed_objects(self, weigher_classes, obj_list, - weighing_properties): - """Return a sorted (descending), normalized list of WeighedObjects.""" - - if not obj_list: - return [] - - weighed_objs = [self.object_class(obj, 0.0) for obj in obj_list] - for weigher_cls in weigher_classes: - weigher = weigher_cls() - weights = weigher.weigh_objects(weighed_objs, weighing_properties) - - # Normalize the weights - weights = normalize(weights, - minval=weigher.minval, - maxval=weigher.maxval) - - for i, weight in enumerate(weights): - obj = weighed_objs[i] - obj.weight += weigher.weight_multiplier() * weight - - return sorted(weighed_objs, key=lambda x: x.weight, reverse=True) diff --git a/mogan/scheduler/client/__init__.py b/mogan/scheduler/client/__init__.py index 54b7e00d..7acc840a 100644 --- a/mogan/scheduler/client/__init__.py +++ b/mogan/scheduler/client/__init__.py @@ -17,6 +17,8 @@ import functools from oslo_utils import importutils +from mogan.scheduler import utils + class LazyLoader(object): @@ -39,9 +41,16 @@ class SchedulerClient(object): """Client library for placing calls to the scheduler.""" def __init__(self): + self.queryclient = LazyLoader(importutils.import_class( + 'mogan.scheduler.client.query.SchedulerQueryClient')) self.reportclient = LazyLoader(importutils.import_class( 'mogan.scheduler.client.report.SchedulerReportClient')) + @utils.retry_select_destinations + def select_destinations(self, context, spec_obj, filter_properties): + return self.queryclient.select_destinations(context, spec_obj, + filter_properties) + def set_inventory_for_provider(self, rp_uuid, rp_name, inv_data, res_class): self.reportclient.set_inventory_for_provider( diff --git a/mogan/scheduler/client/query.py b/mogan/scheduler/client/query.py new file mode 100644 index 00000000..9bcfe0dd --- /dev/null +++ b/mogan/scheduler/client/query.py @@ -0,0 +1,33 @@ +# Copyright (c) 2014 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from mogan.scheduler import rpcapi as scheduler_rpcapi + + +class SchedulerQueryClient(object): + """Client class for querying to the scheduler.""" + + def __init__(self): + self.scheduler_rpcapi = scheduler_rpcapi.SchedulerAPI() + + def select_destinations(self, context, spec_obj, filter_properties): + """Returns destinations(s) best suited for this request_spec and + filter_properties. + + The result should be a list of dicts with 'host', 'nodename' and + 'limits' as keys. + """ + return self.scheduler_rpcapi.select_destinations(context, spec_obj, + filter_properties) diff --git a/mogan/scheduler/client/report.py b/mogan/scheduler/client/report.py index d462e5df..2612b9c0 100644 --- a/mogan/scheduler/client/report.py +++ b/mogan/scheduler/client/report.py @@ -98,7 +98,7 @@ class SchedulerReportClient(object): def __init__(self): # A dict, keyed by the resource provider UUID, of ResourceProvider # objects that will have their inventories and allocations tracked by - # the placement API for the compute host + # the placement API for the node self._resource_providers = {} # A dict, keyed by resource provider UUID, of sets of aggregate UUIDs # the provider is associated with @@ -167,8 +167,6 @@ class SchedulerReportClient(object): url, endpoint_filter=self.ks_filter, raise_exc=False) - # TODO(sbauza): Change that poor interface into passing a rich versioned - # object that would provide the ResourceProvider requirements. @safe_connect def get_filtered_resource_providers(self, filters): """Returns a list of ResourceProviders matching the requirements @@ -421,34 +419,6 @@ class SchedulerReportClient(object): {'placement_req_id': get_placement_request_id(result), 'resource_provider_uuid': rp_uuid, 'generation_id': cur_rp_gen}) - # NOTE(jaypipes): There may be cases when we try to set a - # provider's inventory that results in attempting to delete an - # inventory record for a resource class that has an active - # allocation. We need to catch this particular case and raise an - # exception here instead of returning False, since we should not - # re-try the operation in this case. - # - # A use case for where this can occur is the following: - # - # 1) Provider created for each Ironic baremetal node in Newton - # 2) Inventory records for baremetal node created for VCPU, - # MEMORY_MB and DISK_GB - # 3) A Nova instance consumes the baremetal node and allocation - # records are created for VCPU, MEMORY_MB and DISK_GB matching - # the total amount of those resource on the baremetal node. - # 3) Upgrade to Ocata and now resource tracker wants to set the - # provider's inventory to a single record of resource class - # CUSTOM_IRON_SILVER (or whatever the Ironic node's - # "resource_class" attribute is) - # 4) Scheduler report client sends the inventory list containing a - # single CUSTOM_IRON_SILVER record and placement service - # attempts to delete the inventory records for VCPU, MEMORY_MB - # and DISK_GB. An exception is raised from the placement service - # because allocation records exist for those resource classes, - # and a 409 Conflict is returned to the compute node. We need to - # trigger a delete of the old allocation records and then set - # the new inventory, and then set the allocation record to the - # new CUSTOM_IRON_SILVER record. match = _RE_INV_IN_USE.search(result.text) if match: rc = match.group(1) @@ -517,71 +487,6 @@ class SchedulerReportClient(object): time.sleep(1) return False - @safe_connect - def _delete_inventory(self, rp_uuid): - """Deletes all inventory records for a resource provider with the - supplied UUID. - """ - curr = self._get_inventory_and_update_provider_generation(rp_uuid) - - # Check to see if we need to update placement's view - if not curr.get('inventories', {}): - msg = "No inventory to delete from resource provider %s." - LOG.debug(msg, rp_uuid) - return - - msg = ("Compute node %s reported no inventory but previous " - "inventory was detected. Deleting existing inventory " - "records.") - LOG.info(msg, rp_uuid) - - url = '/resource_providers/%s/inventories' % rp_uuid - cur_rp_gen = self._resource_providers[rp_uuid]['generation'] - payload = { - 'resource_provider_generation': cur_rp_gen, - 'inventories': {}, - } - r = self.put(url, payload) - placement_req_id = get_placement_request_id(r) - if r.status_code == 200: - # Update our view of the generation for next time - updated_inv = r.json() - new_gen = updated_inv['resource_provider_generation'] - - self._resource_providers[rp_uuid]['generation'] = new_gen - msg_args = { - 'rp_uuid': rp_uuid, - 'generation': new_gen, - 'placement_req_id': placement_req_id, - } - LOG.info(('[%(placement_req_id)s] Deleted all inventory for ' - 'resource provider %(rp_uuid)s at generation ' - '%(generation)i'), - msg_args) - return - elif r.status_code == 409: - rc_str = _extract_inventory_in_use(r.text) - if rc_str is not None: - msg = ("[%(placement_req_id)s] We cannot delete inventory " - "%(rc_str)s for resource provider %(rp_uuid)s " - "because the inventory is in use.") - msg_args = { - 'rp_uuid': rp_uuid, - 'rc_str': rc_str, - 'placement_req_id': placement_req_id, - } - LOG.warning(msg, msg_args) - return - - msg = ("[%(placement_req_id)s] Failed to delete inventory for " - "resource provider %(rp_uuid)s. Got error response: %(err)s") - msg_args = { - 'rp_uuid': rp_uuid, - 'err': r.text, - 'placement_req_id': placement_req_id, - } - LOG.error(msg, msg_args) - def set_inventory_for_provider(self, rp_uuid, rp_name, inv_data, resource_class): """Given the UUID of a provider, set the inventory records for the @@ -601,10 +506,7 @@ class SchedulerReportClient(object): # Auto-create custom resource classes coming from a virt driver self._ensure_resource_class(resource_class) - if inv_data: - self._update_inventory(rp_uuid, inv_data) - else: - self._delete_inventory(rp_uuid) + self._update_inventory(rp_uuid, inv_data) @safe_connect def _ensure_resource_class(self, name): @@ -704,8 +606,26 @@ class SchedulerReportClient(object): raise exception.InvalidResourceClass(resource_class=name) @safe_connect - def put_allocations(self, rp_uuid, consumer_uuid, alloc_data): - """Creates allocation records for the supplied instance UUID against + def delete_allocation_for_server(self, uuid): + url = '/allocations/%s' % uuid + r = self.delete(url) + if r: + LOG.info('Deleted allocation for server %s', uuid) + else: + # Check for 404 since we don't need to log a warning if we tried to + # delete something which doesn't actually exist. + if r.status_code != 404: + LOG.warning( + 'Unable to delete allocation for server ' + '%(uuid)s: (%(code)i %(text)s)', + {'uuid': uuid, + 'code': r.status_code, + 'text': r.text}) + + @safe_connect + def put_allocations(self, rp_uuid, consumer_uuid, alloc_data, project_id, + user_id): + """Creates allocation records for the supplied server UUID against the supplied resource provider. :note Currently we only allocate against a single resource provider. @@ -713,9 +633,11 @@ class SchedulerReportClient(object): reality, this will change to allocate against multiple providers. :param rp_uuid: The UUID of the resource provider to allocate against. - :param consumer_uuid: The instance's UUID. + :param consumer_uuid: The server's UUID. :param alloc_data: Dict, keyed by resource class, of amounts to consume. + :param project_id: The project_id associated with the allocations. + :param user_id: The user_id associated with the allocations. :returns: True if the allocations were created, False otherwise. """ payload = { @@ -727,18 +649,51 @@ class SchedulerReportClient(object): 'resources': alloc_data, }, ], + 'project_id': project_id, + 'user_id': user_id, } url = '/allocations/%s' % consumer_uuid - r = self.put(url, payload) + r = self.put(url, payload, version='1.8') + if r.status_code == 406: + # microversion 1.8 not available so try the earlier way + # TODO(melwitt): Remove this when we can be sure all placement + # servers support version 1.8. + payload.pop('project_id') + payload.pop('user_id') + r = self.put(url, payload) if r.status_code != 204: LOG.warning( - 'Unable to submit allocation for instance ' + 'Unable to submit allocation for server ' '%(uuid)s (%(code)i %(text)s)', {'uuid': consumer_uuid, 'code': r.status_code, 'text': r.text}) return r.status_code == 204 + @safe_connect + def delete_resource_provider(self, rp_uuid): + """Deletes the ResourceProvider record for the compute_node. + + :param rp_uuid: The uuid of resource provider being deleted. + """ + url = "/resource_providers/%s" % rp_uuid + resp = self.delete(url) + if resp: + LOG.info("Deleted resource provider %s", rp_uuid) + # clean the caches + self._resource_providers.pop(rp_uuid, None) + self._provider_aggregate_map.pop(rp_uuid, None) + else: + # Check for 404 since we don't need to log a warning if we tried to + # delete something which doesn"t actually exist. + if resp.status_code != 404: + LOG.warning( + "Unable to delete resource provider " + "%(uuid)s: (%(code)i %(text)s)", + {"uuid": rp_uuid, + "code": resp.status_code, + "text": resp.text}) + @safe_connect def get_allocations_for_resource_provider(self, rp_uuid): url = '/resource_providers/%s/allocations' % rp_uuid @@ -747,3 +702,10 @@ class SchedulerReportClient(object): return {} else: return resp.json()['allocations'] + + def delete_allocations_for_resource_provider(self, rp_uuid): + allocations = self.get_allocations_for_resource_provider(rp_uuid) + if allocations: + LOG.info('Deleted allocation for resource provider %s', rp_uuid) + for consumer_id in allocations: + self.delete_allocation_for_server(consumer_id) diff --git a/mogan/scheduler/driver.py b/mogan/scheduler/driver.py index 1e778baa..726179cf 100644 --- a/mogan/scheduler/driver.py +++ b/mogan/scheduler/driver.py @@ -20,7 +20,6 @@ Scheduler base class that all Schedulers should inherit from """ from oslo_config import cfg -from oslo_utils import importutils from mogan.common.i18n import _ @@ -31,10 +30,6 @@ CONF = cfg.CONF class Scheduler(object): """The base class that all Scheduler classes should inherit from.""" - def __init__(self): - self.node_manager = importutils.import_object( - CONF.scheduler.scheduler_node_manager) - def schedule(self, context, request_spec, filter_properties): """Must override schedule method for scheduler to work.""" raise NotImplementedError(_("Must implement schedule")) diff --git a/mogan/scheduler/filter_scheduler.py b/mogan/scheduler/filter_scheduler.py index 20f91db3..463e2f8e 100644 --- a/mogan/scheduler/filter_scheduler.py +++ b/mogan/scheduler/filter_scheduler.py @@ -15,16 +15,16 @@ You can customize this scheduler by specifying your own node Filters and Weighing Functions. """ - from oslo_config import cfg from oslo_log import log as logging -from oslo_serialization import jsonutils from mogan.common import exception from mogan.common.i18n import _ from mogan.common import utils +from mogan import objects +from mogan.scheduler import client from mogan.scheduler import driver -from mogan.scheduler import scheduler_options +from mogan.scheduler import utils as sched_utils CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -34,21 +34,8 @@ class FilterScheduler(driver.Scheduler): """Scheduler that can be used for filtering and weighing.""" def __init__(self, *args, **kwargs): super(FilterScheduler, self).__init__(*args, **kwargs) - self.options = scheduler_options.SchedulerOptions() self.max_attempts = self._max_attempts() - - def _get_configuration_options(self): - """Fetch options dictionary. Broken out for testing.""" - return self.options.get_configuration() - - def populate_filter_properties(self, request_spec, filter_properties): - """Stuff things into filter_properties. - - Can be overridden in a subclass to add more data. - """ - server = request_spec['server_properties'] - filter_properties['availability_zone'] = \ - server.get('availability_zone') + self.reportclient = client.SchedulerClient().reportclient def _max_attempts(self): max_attempts = CONF.scheduler.scheduler_max_attempts @@ -87,7 +74,7 @@ class FilterScheduler(driver.Scheduler): # re-scheduling is disabled. return - server_id = request_spec.get('server_id') + server_id = request_spec.get('server_ids')[0] self._log_server_error(server_id, retry) if retry['num_attempts'] > max_attempts: @@ -97,54 +84,21 @@ class FilterScheduler(driver.Scheduler): {'max_attempts': max_attempts, 'server_id': server_id}) - def _get_weighted_candidates(self, context, request_spec, - filter_properties=None): - """Return a list of nodes that meet required specs. + @staticmethod + def _get_res_cls_filters(request_spec): + flavor_dict = request_spec['flavor'] + resources = dict([(sched_utils.ensure_resource_class_name(res[0]), + int(res[1])) + for res in flavor_dict['resources'].items()]) + return resources - Returned list is ordered by their fitness. - """ - # Since Mogan is using mixed filters from Oslo and it's own, which - # takes 'resource_XX' and 'server_XX' as input respectively, copying - # 'flavor' to 'resource_type' will make both filters happy. - flavor = resource_type = request_spec.get("flavor") - - config_options = self._get_configuration_options() - - if filter_properties is None: - filter_properties = {} - self._populate_retry(filter_properties, request_spec) - - request_spec_dict = jsonutils.to_primitive(request_spec) - - filter_properties.update({'request_spec': request_spec_dict, - 'config_options': config_options, - 'flavor': flavor, - 'resource_type': resource_type}) - - self.populate_filter_properties(request_spec, - filter_properties) - - # Find our local list of acceptable nodes by filtering and - # weighing our options. we virtually consume resources on - # it so subsequent selections can adjust accordingly. - - # Note: remember, we are using an iterator here. So only - # traverse this list once. - nodes = self.node_manager.get_all_node_states(context) - - # Filter local nodes based on requirements ... - nodes = self.node_manager.get_filtered_nodes(nodes, - filter_properties) - if not nodes: + def _get_filtered_nodes(self, request_spec): + query_filters = {'resources': self._get_res_cls_filters(request_spec)} + filtered_nodes = self.reportclient.get_filtered_resource_providers( + query_filters) + if not filtered_nodes: return [] - - LOG.debug("Filtered %(nodes)s", {'nodes': nodes}) - # weighted_node = WeightedNode() ... the best - # node for the job. - weighed_nodes = self.node_manager.get_weighed_nodes(nodes, - filter_properties) - LOG.debug("Weighed %(nodes)s", {'nodes': weighed_nodes}) - return weighed_nodes + return [node['uuid'] for node in filtered_nodes] def schedule(self, context, request_spec, filter_properties=None): @@ -155,20 +109,21 @@ class FilterScheduler(driver.Scheduler): # we need to improve this. @utils.synchronized('schedule') def _schedule(self, context, request_spec, filter_properties): - weighed_nodes = self._get_weighted_candidates( - context, request_spec, filter_properties) - if not weighed_nodes: - LOG.warning('No weighed nodes found for server ' + self._populate_retry(filter_properties, request_spec) + filtered_nodes = self._get_filtered_nodes(request_spec) + if not filtered_nodes: + LOG.warning('No filtered nodes found for server ' 'with properties: %s', request_spec.get('flavor')) - raise exception.NoValidNode(_("No weighed nodes available")) - - dest_nodes = [] - nodes = self._choose_nodes(weighed_nodes, request_spec) - for node in nodes: - node.obj.consume_from_request(context) - dest_nodes.append( - dict(node_uuid=node.obj.node_uuid, ports=node.obj.ports)) + raise exception.NoValidNode(_("No filtered nodes available")) + dest_nodes = self._choose_nodes(filtered_nodes, request_spec) + for server_id, node in zip(request_spec['server_ids'], dest_nodes): + server_obj = objects.Server.get( + context, server_id) + alloc_data = self._get_res_cls_filters(request_spec) + self.reportclient.put_allocations( + node, server_obj.uuid, alloc_data, + server_obj.project_id, server_obj.user_id) return dest_nodes return _schedule(self, context, request_spec, filter_properties) diff --git a/mogan/scheduler/filters/__init__.py b/mogan/scheduler/filters/__init__.py deleted file mode 100644 index 8b9c8bec..00000000 --- a/mogan/scheduler/filters/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Scheduler node filters -""" - -from mogan.scheduler import base_filter - - -class BaseNodeFilter(base_filter.BaseFilter): - """Base class for node filters.""" - def _filter_one(self, obj, filter_properties): - """Return True if the object passes the filter, otherwise False.""" - return self.node_passes(obj, filter_properties) - - def node_passes(self, node_state, filter_properties): - """Return True if the NodeState passes the filter, otherwise False. - - Override this in a subclass. - """ - raise NotImplementedError() - - -class NodeFilterHandler(base_filter.BaseFilterHandler): - def __init__(self, namespace): - super(NodeFilterHandler, self).__init__(BaseNodeFilter, namespace) diff --git a/mogan/scheduler/filters/availability_zone_filter.py b/mogan/scheduler/filters/availability_zone_filter.py deleted file mode 100644 index 1f827166..00000000 --- a/mogan/scheduler/filters/availability_zone_filter.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2011-2012 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from mogan.scheduler import filters - - -class AvailabilityZoneFilter(filters.BaseNodeFilter): - """Filters Nodes by availability zone.""" - - # Availability zones do not change within a request - run_filter_once_per_request = True - - def node_passes(self, node_state, filter_properties): - spec = filter_properties.get('request_spec', {}) - availability_zone = spec.get('availability_zone') - - if availability_zone: - return availability_zone == node_state.availability_zone - return True diff --git a/mogan/scheduler/filters/capabilities_filter.py b/mogan/scheduler/filters/capabilities_filter.py deleted file mode 100644 index eaf6a1df..00000000 --- a/mogan/scheduler/filters/capabilities_filter.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_log import log as logging - -from mogan.scheduler import filters -from mogan.scheduler.filters import extra_specs_ops - -LOG = logging.getLogger(__name__) - - -class CapabilitiesFilter(filters.BaseNodeFilter): - """NodeFilter to work with resource server type records.""" - - def _satisfies_extra_specs(self, capabilities, resource_type): - """Check if capabilities satisfy resource type requirements. - - Check that the capabilities provided by the services satisfy - the extra specs associated with the resource type. - """ - - if not resource_type: - return True - - extra_specs = resource_type.get('extra_specs', []) - if not extra_specs: - return True - - for key, req in extra_specs.items(): - - # Either not scoped format, or in capabilities scope - scope = key.split(':') - - # Ignore scoped (such as vendor-specific) capabilities - if len(scope) > 1 and scope[0] != "capabilities": - continue - # Strip off prefix if spec started with 'capabilities:' - elif scope[0] == "capabilities": - del scope[0] - - cap = capabilities - for index in range(len(scope)): - try: - cap = cap[scope[index]] - except (TypeError, KeyError): - LOG.debug("Node doesn't provide capability '%(cap)s' " % - {'cap': scope[index]}) - return False - - # Make all capability values a list so we can handle lists - cap_list = [cap] if not isinstance(cap, list) else cap - - # Loop through capability values looking for any match - for cap_value in cap_list: - if extra_specs_ops.match(cap_value, req): - break - else: - # Nothing matched, so bail out - LOG.debug('Flavor extra spec requirement ' - '"%(key)s=%(req)s" does not match reported ' - 'capability "%(cap)s"', - {'key': key, 'req': req, 'cap': cap}) - return False - return True - - def node_passes(self, node_state, filter_properties): - """Return a list of nodes that can create resource_type.""" - resource_type = filter_properties.get('resource_type') - if not self._satisfies_extra_specs(node_state.capabilities, - resource_type): - LOG.debug("%(node_state)s fails resource_type extra_specs " - "requirements", {'node_state': node_state}) - return False - return True diff --git a/mogan/scheduler/filters/extra_specs_ops.py b/mogan/scheduler/filters/extra_specs_ops.py deleted file mode 100644 index 24b48a31..00000000 --- a/mogan/scheduler/filters/extra_specs_ops.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import operator - -from oslo_utils import strutils - -# 1. The following operations are supported: -# =, s==, s!=, s>=, s>, s<=, s<, , , , ==, !=, >=, <= -# 2. Note that is handled in a different way below. -# 3. If the first word in the extra_specs is not one of the operators, -# it is ignored. -_op_methods = {'=': lambda x, y: float(x) >= float(y), - '': lambda x, y: y in x, - '': lambda x, y: (strutils.bool_from_string(x) is - strutils.bool_from_string(y)), - '==': lambda x, y: float(x) == float(y), - '!=': lambda x, y: float(x) != float(y), - '>=': lambda x, y: float(x) >= float(y), - '<=': lambda x, y: float(x) <= float(y), - 's==': operator.eq, - 's!=': operator.ne, - 's<': operator.lt, - 's<=': operator.le, - 's>': operator.gt, - 's>=': operator.ge} - - -def match(value, req): - if req is None: - if value is None: - return True - else: - return False - words = req.split() - - op = method = None - if words: - op = words.pop(0) - method = _op_methods.get(op) - - if op != '' and not method: - return value == req - - if value is None: - return False - - if op == '': # Ex: v1 v2 v3 - while True: - if words.pop(0) == value: - return True - if not words: - break - op = words.pop(0) # remove a keyword - if not words: - break - return False - - try: - if words and method(value, words[0]): - return True - except ValueError: - pass - - return False diff --git a/mogan/scheduler/filters/flavor_filter.py b/mogan/scheduler/filters/flavor_filter.py deleted file mode 100644 index f22d79ce..00000000 --- a/mogan/scheduler/filters/flavor_filter.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2016 Huawei Technologies Co.,LTD. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from mogan.scheduler import filters - - -class FlavorFilter(filters.BaseNodeFilter): - """Filters Nodes by server type.""" - - # Flavors do not change within a request - run_filter_once_per_request = True - - def node_passes(self, node_state, filter_properties): - spec = filter_properties.get('request_spec', {}) - flavor = spec.get('flavor', {}) - type_name = flavor.get('name') - - if type_name: - return type_name == node_state.flavor - return True diff --git a/mogan/scheduler/filters/json_filter.py b/mogan/scheduler/filters/json_filter.py deleted file mode 100644 index a94da852..00000000 --- a/mogan/scheduler/filters/json_filter.py +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright (c) 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import operator - -from oslo_serialization import jsonutils -import six - -from mogan.scheduler import filters - - -class JsonFilter(filters.BaseNodeFilter): - """Node Filter to allow simple JSON-based grammar for selecting nodes.""" - def _op_compare(self, args, op): - """Compare first item of args with the rest using specified operator. - - Returns True if the specified operator can successfully - compare the first item in the args with all the rest. Will - return False if only one item is in the list. - """ - if len(args) < 2: - return False - if op is operator.contains: - bad = args[0] not in args[1:] - else: - bad = [arg for arg in args[1:] - if not op(args[0], arg)] - return not bool(bad) - - def _equals(self, args): - """First term is == all the other terms.""" - return self._op_compare(args, operator.eq) - - def _less_than(self, args): - """First term is < all the other terms.""" - return self._op_compare(args, operator.lt) - - def _greater_than(self, args): - """First term is > all the other terms.""" - return self._op_compare(args, operator.gt) - - def _in(self, args): - """First term is in set of remaining terms.""" - return self._op_compare(args, operator.contains) - - def _less_than_equal(self, args): - """First term is <= all the other terms.""" - return self._op_compare(args, operator.le) - - def _greater_than_equal(self, args): - """First term is >= all the other terms.""" - return self._op_compare(args, operator.ge) - - def _not(self, args): - """Flip each of the arguments.""" - return [not arg for arg in args] - - def _or(self, args): - """True if any arg is True.""" - return any(args) - - def _and(self, args): - """True if all args are True.""" - return all(args) - - commands = { - '=': _equals, - '<': _less_than, - '>': _greater_than, - 'in': _in, - '<=': _less_than_equal, - '>=': _greater_than_equal, - 'not': _not, - 'or': _or, - 'and': _and, - } - - def _parse_string(self, string, node_state): - """Parse capability lookup strings. - - Strings prefixed with $ are capability lookups in the - form '$variable' where 'variable' is an attribute in the - NodeState class. If $variable is a dictionary, you may - use: $variable.dictkey - """ - if not string: - return None - if not string.startswith("$"): - return string - - path = string[1:].split(".") - obj = getattr(node_state, path[0], None) - if obj is None: - return None - for item in path[1:]: - obj = obj.get(item) - if obj is None: - return None - return obj - - def _process_filter(self, query, node_state): - """Recursively parse the query structure.""" - if not query: - return True - cmd = query[0] - method = self.commands[cmd] - cooked_args = [] - for arg in query[1:]: - if isinstance(arg, list): - arg = self._process_filter(arg, node_state) - elif isinstance(arg, six.string_types): - arg = self._parse_string(arg, node_state) - if arg is not None: - cooked_args.append(arg) - result = method(self, cooked_args) - return result - - def node_passes(self, node_state, filter_properties): - """Return a list of nodes that can fulfill query requirements.""" - try: - query = filter_properties['scheduler_hints']['query'] - except KeyError: - query = None - if not query: - return True - - # NOTE(comstud): Not checking capabilities or service for - # enabled/disabled so that a provided json filter can decide - - result = self._process_filter(jsonutils.loads(query), node_state) - if isinstance(result, list): - # If any succeeded, include the node - result = any(result) - if result: - # Filter it out. - return True - return False diff --git a/mogan/scheduler/filters/ports_filter.py b/mogan/scheduler/filters/ports_filter.py deleted file mode 100644 index 416212d0..00000000 --- a/mogan/scheduler/filters/ports_filter.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2016 Huawei Technologies Co.,LTD. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_log import log as logging - -from mogan.scheduler import filters - -LOG = logging.getLogger(__name__) - - -class PortsFilter(filters.BaseNodeFilter): - """NodeFilter to work with resource server type records.""" - - def _satisfies_networks(self, ports, networks): - """Check if ports satisfy networks requirements. - - Check that the ports provided by the nodes satisfy - the networks associated with the request spec. - """ - - if not networks: - return True - - if len(ports) < len(networks): - return False - - return True - - def node_passes(self, node_state, filter_properties): - """Return a list of nodes that can create resource_type.""" - spec = filter_properties.get('request_spec', {}) - props = spec.get('server_properties', {}) - networks = props.get('networks') - if not self._satisfies_networks(node_state.ports, networks): - LOG.debug("%(node_state)s fails network ports " - "requirements", {'node_state': node_state}) - return False - return True diff --git a/mogan/scheduler/manager.py b/mogan/scheduler/manager.py index 94cff717..21101a5a 100644 --- a/mogan/scheduler/manager.py +++ b/mogan/scheduler/manager.py @@ -15,14 +15,13 @@ import eventlet import oslo_messaging as messaging -from oslo_service import periodic_task from oslo_utils import importutils from mogan.common import exception from mogan.conf import CONF -class SchedulerManager(periodic_task.PeriodicTasks): +class SchedulerManager(object): """Mogan Scheduler manager main class.""" RPC_API_VERSION = '1.0' @@ -30,7 +29,7 @@ class SchedulerManager(periodic_task.PeriodicTasks): target = messaging.Target(version=RPC_API_VERSION) def __init__(self, topic, host=None): - super(SchedulerManager, self).__init__(CONF) + super(SchedulerManager, self).__init__() self.host = host or CONF.host self.topic = topic scheduler_driver = CONF.scheduler.scheduler_driver @@ -53,6 +52,3 @@ class SchedulerManager(periodic_task.PeriodicTasks): def del_host(self): pass - - def periodic_tasks(self, context, raise_on_error=False): - return self.run_periodic_tasks(context, raise_on_error=raise_on_error) diff --git a/mogan/scheduler/node_manager.py b/mogan/scheduler/node_manager.py deleted file mode 100644 index b5816b95..00000000 --- a/mogan/scheduler/node_manager.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright (c) 2011 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Manage nodes. -""" - -from oslo_config import cfg -from oslo_log import log as logging -from oslo_utils import importutils - -from mogan.common import exception -from mogan import objects -from mogan.scheduler import filters - - -CONF = cfg.CONF -LOG = logging.getLogger(__name__) - - -class NodeState(object): - """Mutable and immutable information tracked for a Ironic node.""" - - def __init__(self, node): - self.node_uuid = node.node_uuid - self.capabilities = node.extra_specs - self.availability_zone = node.availability_zone \ - or CONF.engine.default_availability_zone - self.flavor = node.resource_class - self.ports = node.ports - - def consume_from_request(self, context): - """Consume the compute node.""" - objects.ComputeNode.consume_node(context, self.node_uuid) - - def __repr__(self): - return "" % (self.node_uuid, self.flavor) - - -class NodeManager(object): - """Base NodeManager class.""" - - node_state_cls = NodeState - - def __init__(self): - self.filter_handler = filters.NodeFilterHandler( - 'mogan.scheduler.filters') - self.filter_classes = self.filter_handler.get_all_classes() - self.weight_handler = importutils.import_object( - CONF.scheduler.scheduler_weight_handler, - 'mogan.scheduler.weights') - self.weight_classes = self.weight_handler.get_all_classes() - - def _choose_node_filters(self, filter_cls_names): - """Return a list of available filter names. - - This function checks input filter names against a predefined set - of acceptable filterss (all loaded filters). If input is None, - it uses CONF.scheduler_default_filters instead. - """ - if filter_cls_names is None: - filter_cls_names = CONF.scheduler.scheduler_default_filters - if not isinstance(filter_cls_names, (list, tuple)): - filter_cls_names = [filter_cls_names] - good_filters = [] - bad_filters = [] - for filter_name in filter_cls_names: - found_class = False - for cls in self.filter_classes: - if cls.__name__ == filter_name: - found_class = True - good_filters.append(cls) - break - if not found_class: - bad_filters.append(filter_name) - if bad_filters: - raise exception.SchedulerNodeFilterNotFound( - filter_name=", ".join(bad_filters)) - return good_filters - - def _choose_node_weighers(self, weight_cls_names): - """Return a list of available weigher names. - - This function checks input weigher names against a predefined set - of acceptable weighers (all loaded weighers). If input is None, - it uses CONF.scheduler_default_weighers instead. - """ - if weight_cls_names is None: - weight_cls_names = CONF.scheduler.scheduler_default_weighers - if not isinstance(weight_cls_names, (list, tuple)): - weight_cls_names = [weight_cls_names] - - good_weighers = [] - bad_weighers = [] - for weigher_name in weight_cls_names: - found_class = False - for cls in self.weight_classes: - if cls.__name__ == weigher_name: - good_weighers.append(cls) - found_class = True - break - if not found_class: - bad_weighers.append(weigher_name) - if bad_weighers: - raise exception.SchedulerNodeWeigherNotFound( - weigher_name=", ".join(bad_weighers)) - return good_weighers - - def get_filtered_nodes(self, nodes, filter_properties, - filter_class_names=None): - """Filter nodes and return only ones passing all filters.""" - filter_classes = self._choose_node_filters(filter_class_names) - return self.filter_handler.get_filtered_objects(filter_classes, - nodes, - filter_properties) - - def get_weighed_nodes(self, nodes, weight_properties, - weigher_class_names=None): - """Weigh the nodes.""" - weigher_classes = self._choose_node_weighers(weigher_class_names) - return self.weight_handler.get_weighed_objects(weigher_classes, - nodes, - weight_properties) - - def get_all_node_states(self, context): - """Returns a list of all the nodes the NodeManager knows about.""" - - nodes = objects.ComputeNodeList.get_all_available(context) - node_states = [] - for node in nodes: - node_state = self.node_state_cls(node) - node_states.append(node_state) - - return node_states diff --git a/mogan/scheduler/scheduler_options.py b/mogan/scheduler/scheduler_options.py deleted file mode 100644 index 11cb5032..00000000 --- a/mogan/scheduler/scheduler_options.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) 2011 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -SchedulerOptions monitors a local .json file for changes and loads -it if needed. This file is converted to a data structure and passed -into the filtering and weighing functions which can use it for -dynamic configuration. -""" - -import datetime -import json -import os - -from oslo_config import cfg -from oslo_log import log as logging -from oslo_utils import timeutils - - -CONF = cfg.CONF -LOG = logging.getLogger(__name__) - - -class SchedulerOptions(object): - """SchedulerOptions monitors a local .json file for changes. - - The file is reloaded if needed and converted to a data structure and - passed into the filtering and weighing functions which can use it - for dynamic configuration. - """ - - def __init__(self): - super(SchedulerOptions, self).__init__() - self.data = {} - self.last_modified = None - self.last_checked = None - - def _get_file_handle(self, filename): - """Get file handle. Broken out for testing.""" - return open(filename) - - def _get_file_timestamp(self, filename): - """Get the last modified datetime. Broken out for testing.""" - try: - return os.path.getmtime(filename) - except os.error: - LOG.exception("Could not stat scheduler options file " - "%(filename)s.", - {'filename': filename}) - raise - - def _load_file(self, handle): - """Decode the JSON file. Broken out for testing.""" - try: - return json.load(handle) - except ValueError: - LOG.exception("Could not decode scheduler options.") - return {} - - def _get_time_now(self): - """Get current UTC. Broken out for testing.""" - return timeutils.utcnow() - - def get_configuration(self, filename=None): - """Check the json file for changes and load it if needed.""" - if not filename: - filename = CONF.scheduler.scheduler_json_config_location - if not filename: - return self.data - if self.last_checked: - now = self._get_time_now() - if now - self.last_checked < datetime.timedelta(minutes=5): - return self.data - - last_modified = self._get_file_timestamp(filename) - if (not last_modified or not self.last_modified or - last_modified > self.last_modified): - self.data = self._load_file(self._get_file_handle(filename)) - self.last_modified = last_modified - if not self.data: - self.data = {} - - return self.data diff --git a/mogan/scheduler/weights/__init__.py b/mogan/scheduler/weights/__init__.py deleted file mode 100644 index a325e126..00000000 --- a/mogan/scheduler/weights/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Scheduler node weights -""" - -from mogan.scheduler import base_weight - - -class WeighedNode(base_weight.WeighedObject): - def to_dict(self): - return { - 'weight': self.weight, - 'node': self.obj.node_uuid, - } - - def __repr__(self): - return ("WeighedNode [node: %s, weight: %s]" % - (self.obj.node_uuid, self.weight)) - - -class BaseNodeWeigher(base_weight.BaseWeigher): - """Base class for node weights.""" - pass - - -class OrderedNodeWeightHandler(base_weight.BaseWeightHandler): - object_class = WeighedNode - - def __init__(self, namespace): - super(OrderedNodeWeightHandler, self).__init__(BaseNodeWeigher, - namespace) diff --git a/mogan/scheduler/weights/port.py b/mogan/scheduler/weights/port.py deleted file mode 100644 index 7209f0c5..00000000 --- a/mogan/scheduler/weights/port.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2017 Huawei Technologies Co.,LTD. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Port Weigher. Weigh nodes by their ports quantity. - -The default is to preferably choose nodes with less ports. If you prefer -choosing more ports nodes, you can set the 'port_weight_multiplier' option -to a positive number and the weighing has the opposite effect of the default. -""" - -from oslo_config import cfg - -from mogan.scheduler import weights - -CONF = cfg.CONF - - -class PortWeigher(weights.BaseNodeWeigher): - minval = 0 - - def weight_multiplier(self): - """Override the weight multiplier.""" - return CONF.scheduler.port_weight_multiplier - - def _weigh_object(self, node_state, weight_properties): - """Higher weights win. We want to choose less ports node to be the - default. - """ - return len(node_state.ports) diff --git a/mogan/tests/unit/engine/flows/test_create_server_flow.py b/mogan/tests/unit/engine/flows/test_create_server_flow.py index d17e37fe..5a541b57 100644 --- a/mogan/tests/unit/engine/flows/test_create_server_flow.py +++ b/mogan/tests/unit/engine/flows/test_create_server_flow.py @@ -36,18 +36,16 @@ class CreateServerFlowTestCase(base.TestCase): def test_create_network_task_execute(self, mock_build_networks, mock_save): fake_engine_manager = mock.MagicMock() fake_requested_networks = mock.MagicMock() - fake_ports = mock.MagicMock() task = create_server.BuildNetworkTask(fake_engine_manager) server_obj = obj_utils.get_test_server(self.ctxt) mock_build_networks.return_value = None mock_save.return_value = None task.execute( - self.ctxt, server_obj, fake_requested_networks, fake_ports) + self.ctxt, server_obj, fake_requested_networks) mock_build_networks.assert_called_once_with(self.ctxt, server_obj, - fake_requested_networks, - fake_ports) + fake_requested_networks) @mock.patch.object(IronicDriver, 'spawn') def test_create_server_task_execute(self, mock_spawn): diff --git a/mogan/tests/unit/scheduler/__init__.py b/mogan/tests/unit/scheduler/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mogan/tests/unit/scheduler/fakes.py b/mogan/tests/unit/scheduler/fakes.py deleted file mode 100644 index 6cd16d01..00000000 --- a/mogan/tests/unit/scheduler/fakes.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2016 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -""" -Fakes For Scheduler tests. -""" - -from mogan.scheduler import filter_scheduler -from mogan.scheduler import node_manager - - -class FakeFilterScheduler(filter_scheduler.FilterScheduler): - def __init__(self, *args, **kwargs): - super(FakeFilterScheduler, self).__init__(*args, **kwargs) - self.node_manager = node_manager.NodeManager() - - -class FakeNodeState(node_manager.NodeState): - def __init__(self, node, attribute_dict): - super(FakeNodeState, self).__init__(node) - for (key, val) in attribute_dict.items(): - setattr(self, key, val) diff --git a/mogan/tests/unit/scheduler/test_base_filter.py b/mogan/tests/unit/scheduler/test_base_filter.py deleted file mode 100644 index 55d18beb..00000000 --- a/mogan/tests/unit/scheduler/test_base_filter.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright (c) 2016 OpenStack Foundation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock - -from mogan.scheduler import base_filter -from mogan.tests import base as test - - -class TestBaseFilter(test.TestCase): - - def setUp(self): - super(TestBaseFilter, self).setUp() - self.filter = base_filter.BaseFilter() - - def test_filter_one_is_called(self): - filters = [1, 2, 3, 4] - filter_properties = {'x': 'y'} - - self.filter._filter_one = mock.Mock() - self.filter._filter_one.side_effect = [False, True, True, False] - calls = [mock.call(i, filter_properties) for i in filters] - - result = list(self.filter.filter_all(filters, filter_properties)) - self.assertEqual([2, 3], result) - self.filter._filter_one.assert_has_calls(calls) - - -class FakeExtension(object): - - def __init__(self, plugin): - self.plugin = plugin - - -class BaseFakeFilter(base_filter.BaseFilter): - pass - - -class FakeFilter1(BaseFakeFilter): - """Derives from BaseFakeFilter and has a fake entry point defined. - - Entry point is returned by fake ExtensionManager. - Should be included in the output of all_classes. - """ - pass - - -class FakeFilter2(BaseFakeFilter): - """Derives from BaseFakeFilter but has no entry point. - - Should be not included in all_classes. - """ - pass - - -class FakeFilter3(base_filter.BaseFilter): - """Does not derive from BaseFakeFilter. - - Should not be included. - """ - pass - - -class FakeFilter4(BaseFakeFilter): - """Derives from BaseFakeFilter and has an entry point. - - Should be included. - """ - pass - - -class FakeFilter5(BaseFakeFilter): - """Derives from BaseFakeFilter but has no entry point. - - Should not be included. - """ - run_filter_once_per_request = True - pass - - -class FilterA(base_filter.BaseFilter): - def filter_all(self, list_objs, filter_properties): - # return all but the first object - return list_objs[1:] - - -class FilterB(base_filter.BaseFilter): - def filter_all(self, list_objs, filter_properties): - # return an empty list - return None - - -class FakeExtensionManager(list): - - def __init__(self, namespace): - classes = [FakeFilter1, FakeFilter3, FakeFilter4] - exts = map(FakeExtension, classes) - super(FakeExtensionManager, self).__init__(exts) - self.namespace = namespace - - -class TestBaseFilterHandler(test.TestCase): - - def setUp(self): - super(TestBaseFilterHandler, self).setUp() - self.mock_object(base_filter.base_handler.extension, - 'ExtensionManager', FakeExtensionManager) - self.handler = base_filter.BaseFilterHandler(BaseFakeFilter, - 'fake_filters') - - def test_get_all_classes(self): - # In order for a FakeFilter to be returned by get_all_classes, it has - # to comply with these rules: - # * It must be derived from BaseFakeFilter - # AND - # * It must have a python entrypoint assigned (returned by - # FakeExtensionManager) - expected = [FakeFilter1, FakeFilter4] - result = self.handler.get_all_classes() - self.assertEqual(expected, result) - - def _get_filtered_objects(self, filter_classes, index=0): - filter_objs_initial = [1, 2, 3, 4] - filter_properties = {'x': 'y'} - return self.handler.get_filtered_objects(filter_classes, - filter_objs_initial, - filter_properties, - index) - - @mock.patch.object(FakeFilter4, 'filter_all') - @mock.patch.object(FakeFilter3, 'filter_all', return_value=None) - def test_get_filtered_objects_return_none(self, fake3_filter_all, - fake4_filter_all): - filter_classes = [FakeFilter1, FakeFilter2, FakeFilter3, FakeFilter4] - result = self._get_filtered_objects(filter_classes) - self.assertIsNone(result) - fake4_filter_all.assert_not_called() - - def test_get_filtered_objects(self): - filter_objs_expected = [1, 2, 3, 4] - filter_classes = [FakeFilter1, FakeFilter2, FakeFilter3, FakeFilter4] - result = self._get_filtered_objects(filter_classes) - self.assertEqual(filter_objs_expected, result) - - def test_get_filtered_objects_with_filter_run_once(self): - filter_objs_expected = [1, 2, 3, 4] - filter_classes = [FakeFilter5] - - with mock.patch.object(FakeFilter5, 'filter_all', - return_value=filter_objs_expected - ) as fake5_filter_all: - result = self._get_filtered_objects(filter_classes) - self.assertEqual(filter_objs_expected, result) - self.assertEqual(1, fake5_filter_all.call_count) - - result = self._get_filtered_objects(filter_classes, index=1) - self.assertEqual(filter_objs_expected, result) - self.assertEqual(1, fake5_filter_all.call_count) - - result = self._get_filtered_objects(filter_classes, index=2) - self.assertEqual(filter_objs_expected, result) - self.assertEqual(1, fake5_filter_all.call_count) diff --git a/mogan/tests/unit/scheduler/test_node_manager.py b/mogan/tests/unit/scheduler/test_node_manager.py deleted file mode 100644 index b33cf10c..00000000 --- a/mogan/tests/unit/scheduler/test_node_manager.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2016 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Tests For NodeManager -""" - -import mock -from oslo_context import context -from oslo_versionedobjects import base as object_base - -from mogan.common import exception -from mogan.objects import compute_port -from mogan.scheduler import filters -from mogan.scheduler import node_manager -from mogan.scheduler.node_manager import NodeState -from mogan.tests import base as test -from mogan.tests.unit.objects import utils as obj_utils - - -class FakeFilterClass1(filters.BaseNodeFilter): - def node_passes(self, node_state, filter_properties): - pass - - -class FakeFilterClass2(filters.BaseNodeFilter): - def node_passes(self, node_state, filter_properties): - pass - - -class NodeManagerTestCase(test.TestCase): - """Test case for NodeManager class.""" - - def setUp(self): - super(NodeManagerTestCase, self).setUp() - self.ctxt = context.get_admin_context() - self.node_manager = node_manager.NodeManager() - - fake_node = obj_utils.get_test_compute_node(self.ctxt) - fake_ports = object_base.obj_make_list( - self.ctxt, compute_port.ComputePortList(self.ctxt), - compute_port.ComputePort, []) - fake_node.ports = fake_ports - self.fake_nodes = [NodeState(fake_node)] - - def test_choose_node_filters_not_found(self): - self.override_config('scheduler_default_filters', 'FakeFilterClass3', - 'scheduler') - self.node_manager.filter_classes = [FakeFilterClass1, - FakeFilterClass2] - self.assertRaises(exception.SchedulerNodeFilterNotFound, - self.node_manager._choose_node_filters, None) - - def test_choose_node_filters(self): - self.override_config('scheduler_default_filters', 'FakeFilterClass2', - group='scheduler') - self.node_manager.filter_classes = [FakeFilterClass1, - FakeFilterClass2] - - # Test returns 1 correct filter class - filter_classes = self.node_manager._choose_node_filters(None) - self.assertEqual(1, len(filter_classes)) - self.assertEqual('FakeFilterClass2', filter_classes[0].__name__) - - @mock.patch('mogan.scheduler.node_manager.NodeManager.' - '_choose_node_filters') - def test_get_filtered_nodes(self, _mock_choose_node_filters): - filter_class = FakeFilterClass1 - mock_func = mock.Mock() - mock_func.return_value = True - filter_class._filter_one = mock_func - _mock_choose_node_filters.return_value = [filter_class] - - fake_properties = {'moo': 1, 'cow': 2} - expected = [] - for fake_node in self.fake_nodes: - expected.append(mock.call(fake_node, fake_properties)) - - result = self.node_manager.get_filtered_nodes(self.fake_nodes, - fake_properties) - self.assertEqual(expected, mock_func.call_args_list) - self.assertEqual(set(self.fake_nodes), set(result)) diff --git a/mogan/tests/unit/scheduler/test_rpcapi.py b/mogan/tests/unit/scheduler/test_rpcapi.py deleted file mode 100644 index 034b0b77..00000000 --- a/mogan/tests/unit/scheduler/test_rpcapi.py +++ /dev/null @@ -1,94 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Unit Tests for :py:class:`mogan.scheduler.rpcapi.SchedulerAPI`. -""" - -import copy - -import mock -from oslo_config import cfg -from oslo_messaging import _utils as messaging_utils - -from mogan.scheduler import manager as scheduler_manager -from mogan.scheduler import rpcapi as scheduler_rpcapi -from mogan.tests import base as tests_base -from mogan.tests.unit.db import base - -CONF = cfg.CONF - - -class SchedulerRPCAPITestCase(tests_base.TestCase): - - def test_versions_in_sync(self): - self.assertEqual( - scheduler_manager.SchedulerManager.RPC_API_VERSION, - scheduler_rpcapi.SchedulerAPI.RPC_API_VERSION) - - -class RPCAPITestCase(base.DbTestCase): - - def _test_rpcapi(self, method, rpc_method, **kwargs): - rpcapi = scheduler_rpcapi.SchedulerAPI(topic='fake-topic') - - expected_retval = 'hello world' if rpc_method == 'call' else None - - expected_topic = 'fake-topic' - - target = { - "topic": expected_topic, - "server": CONF.host, - "version": kwargs.pop('version', rpcapi.RPC_API_VERSION) - } - expected_msg = copy.deepcopy(kwargs) - - self.fake_args = None - self.fake_kwargs = None - - def _fake_can_send_version_method(version): - return messaging_utils.version_is_compatible( - rpcapi.RPC_API_VERSION, version) - - def _fake_prepare_method(*args, **kwargs): - for kwd in kwargs: - self.assertEqual(kwargs[kwd], target[kwd]) - return rpcapi.client - - def _fake_rpc_method(*args, **kwargs): - self.fake_args = args - self.fake_kwargs = kwargs - if expected_retval: - return expected_retval - - with mock.patch.object(rpcapi.client, - "can_send_version") as mock_can_send_version: - mock_can_send_version.side_effect = _fake_can_send_version_method - with mock.patch.object(rpcapi.client, "prepare") as mock_prepared: - mock_prepared.side_effect = _fake_prepare_method - - with mock.patch.object(rpcapi.client, - rpc_method) as mock_method: - mock_method.side_effect = _fake_rpc_method - retval = getattr(rpcapi, method)(self.context, **kwargs) - self.assertEqual(retval, expected_retval) - expected_args = [self.context, method, expected_msg] - for arg, expected_arg in zip(self.fake_args, - expected_args): - self.assertEqual(arg, expected_arg) - - def test_select_destinations(self): - self._test_rpcapi('select_destinations', - 'call', - version='1.0', - request_spec=None, - filter_properties=None) diff --git a/mogan/tests/unit/scheduler/test_scheduler_options.py b/mogan/tests/unit/scheduler/test_scheduler_options.py deleted file mode 100644 index 30297a36..00000000 --- a/mogan/tests/unit/scheduler/test_scheduler_options.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2016 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Tests For PickledScheduler. -""" - -import datetime - -from oslo_serialization import jsonutils -import six - -from mogan.scheduler import scheduler_options -from mogan.tests import base as test - - -class FakeSchedulerOptions(scheduler_options.SchedulerOptions): - def __init__(self, last_checked, now, file_old, file_now, data, filedata): - super(FakeSchedulerOptions, self).__init__() - # Change internals ... - self.last_modified = file_old - self.last_checked = last_checked - self.data = data - - # For overrides ... - self._time_now = now - self._file_now = file_now - self._file_data = filedata - - self.file_was_loaded = False - - def _get_file_timestamp(self, filename): - return self._file_now - - def _get_file_handle(self, filename): - self.file_was_loaded = True - return six.StringIO(self._file_data) - - def _get_time_now(self): - return self._time_now - - -class SchedulerOptionsTestCase(test.TestCase): - def test_get_configuration_first_time_no_flag(self): - last_checked = None - now = datetime.datetime(2012, 1, 1, 1, 1, 1) - file_old = None - file_now = datetime.datetime(2012, 1, 1, 1, 1, 1) - - data = dict(a=1, b=2, c=3) - jdata = jsonutils.dumps(data) - - fake = FakeSchedulerOptions(last_checked, now, file_old, file_now, - {}, jdata) - self.assertEqual({}, fake.get_configuration()) - self.assertFalse(fake.file_was_loaded) - - def test_get_configuration_first_time_empty_file(self): - last_checked = None - now = datetime.datetime(2012, 1, 1, 1, 1, 1) - file_old = None - file_now = datetime.datetime(2012, 1, 1, 1, 1, 1) - - jdata = "" - - fake = FakeSchedulerOptions(last_checked, now, file_old, file_now, - {}, jdata) - self.assertEqual({}, fake.get_configuration('foo.json')) - self.assertTrue(fake.file_was_loaded) - - def test_get_configuration_first_time_happy_day(self): - last_checked = None - now = datetime.datetime(2012, 1, 1, 1, 1, 1) - file_old = None - file_now = datetime.datetime(2012, 1, 1, 1, 1, 1) - - data = dict(a=1, b=2, c=3) - jdata = jsonutils.dumps(data) - - fake = FakeSchedulerOptions(last_checked, now, file_old, file_now, - {}, jdata) - self.assertEqual(data, fake.get_configuration('foo.json')) - self.assertTrue(fake.file_was_loaded) - - def test_get_configuration_second_time_no_change(self): - last_checked = datetime.datetime(2011, 1, 1, 1, 1, 1) - now = datetime.datetime(2012, 1, 1, 1, 1, 1) - file_old = datetime.datetime(2012, 1, 1, 1, 1, 1) - file_now = datetime.datetime(2012, 1, 1, 1, 1, 1) - - data = dict(a=1, b=2, c=3) - jdata = jsonutils.dumps(data) - - fake = FakeSchedulerOptions(last_checked, now, file_old, file_now, - data, jdata) - self.assertEqual(data, fake.get_configuration('foo.json')) - self.assertFalse(fake.file_was_loaded) - - def test_get_configuration_second_time_too_fast(self): - last_checked = datetime.datetime(2011, 1, 1, 1, 1, 1) - now = datetime.datetime(2011, 1, 1, 1, 1, 2) - file_old = datetime.datetime(2012, 1, 1, 1, 1, 1) - file_now = datetime.datetime(2013, 1, 1, 1, 1, 1) - - old_data = dict(a=1, b=2, c=3) - data = dict(a=11, b=12, c=13) - jdata = jsonutils.dumps(data) - - fake = FakeSchedulerOptions(last_checked, now, file_old, file_now, - old_data, jdata) - self.assertEqual(old_data, fake.get_configuration('foo.json')) - self.assertFalse(fake.file_was_loaded) - - def test_get_configuration_second_time_change(self): - last_checked = datetime.datetime(2011, 1, 1, 1, 1, 1) - now = datetime.datetime(2012, 1, 1, 1, 1, 1) - file_old = datetime.datetime(2012, 1, 1, 1, 1, 1) - file_now = datetime.datetime(2013, 1, 1, 1, 1, 1) - - old_data = dict(a=1, b=2, c=3) - data = dict(a=11, b=12, c=13) - jdata = jsonutils.dumps(data) - - fake = FakeSchedulerOptions(last_checked, now, file_old, file_now, - old_data, jdata) - self.assertEqual(data, fake.get_configuration('foo.json')) - self.assertTrue(fake.file_was_loaded) diff --git a/mogan/tests/unit/scheduler/test_weights.py b/mogan/tests/unit/scheduler/test_weights.py deleted file mode 100644 index fab12aa6..00000000 --- a/mogan/tests/unit/scheduler/test_weights.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2016 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Tests For Scheduler weights. -""" - -from mogan.scheduler import base_weight -from mogan.tests import base as test - - -class TestWeightHandler(test.TestCase): - def test_no_multiplier(self): - class FakeWeigher(base_weight.BaseWeigher): - def _weigh_object(self, *args, **kwargs): - pass - - self.assertEqual(1.0, - FakeWeigher().weight_multiplier()) - - def test_no_weight_object(self): - class FakeWeigher(base_weight.BaseWeigher): - def weight_multiplier(self, *args, **kwargs): - pass - self.assertRaises(TypeError, - FakeWeigher) - - def test_normalization(self): - # weight_list, expected_result, minval, maxval - map_ = ( - ((), (), None, None), - ((0.0, 0.0), (0.0, 0.0), None, None), - ((1.0, 1.0), (0.0, 0.0), None, None), - - ((20.0, 50.0), (0.0, 1.0), None, None), - ((20.0, 50.0), (0.0, 0.375), None, 100.0), - ((20.0, 50.0), (0.4, 1.0), 0.0, None), - ((20.0, 50.0), (0.2, 0.5), 0.0, 100.0), - ) - for seq, result, minval, maxval in map_: - ret = base_weight.normalize(seq, minval=minval, maxval=maxval) - self.assertEqual(result, tuple(ret)) diff --git a/setup.cfg b/setup.cfg index a87caca8..353eee24 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,15 +24,6 @@ packages = mogan [entry_points] -mogan.scheduler.filters = - AvailabilityZoneFilter = mogan.scheduler.filters.availability_zone_filter:AvailabilityZoneFilter - FlavorFilter = mogan.scheduler.filters.flavor_filter:FlavorFilter - CapabilitiesFilter = mogan.scheduler.filters.capabilities_filter:CapabilitiesFilter - PortsFilter = mogan.scheduler.filters.ports_filter:PortsFilter - JsonFilter = mogan.scheduler.filters.json_filter:JsonFilter -mogan.scheduler.weights = - PortWeigher = mogan.scheduler.weights.port:PortWeigher - oslo.config.opts = mogan = mogan.conf.opts:list_opts