From 47759202a844b5930da78332431b5681e0d2fe06 Mon Sep 17 00:00:00 2001 From: Jean-Emile DARTOIS Date: Thu, 7 Jan 2016 16:54:15 +0100 Subject: [PATCH] Add a dynamic loading of the Watcher Planner implementation In watcher, an audit generates a set of actions which aims at achieving a given goal (lower energy consumption, ...). It is possible to configure different strategies in order to achieve each goal. Each strategy is written as a Python class which produces a set of actions. Today, the set of possible actions is fixed for a given version of Watcher and enables optimization algorithms to include actions such as instance migration, changing hypervisor state, changing power state (ACPI level, ...). The objective of this patchset is to give the ability to extend the default set of planner algorithms currently available in Watcher using Stevedore. The doc need to explain how create a new planner. DocImpact Partially implements: blueprint watcher-add-actions-via-conf Change-Id: I2fd73f8c4a457ee391d764a7a3f494deecd2634f --- etc/watcher/watcher.conf.sample | 38 ++++++------- setup.cfg | 3 ++ watcher/decision_engine/audit/default.py | 14 +++-- watcher/decision_engine/planner/default.py | 6 +-- watcher/decision_engine/planner/manager.py | 53 +++++++++++++++++++ watcher/opts.py | 4 +- .../tests/decision_engine/planner/__init__.py | 0 .../{ => planner}/test_default_planner.py | 10 ++-- .../planner/test_default_planner_loader.py | 29 ++++++++++ .../planner/test_planner_manager.py | 28 ++++++++++ .../loading/test_default_strategy_loader.py | 4 ++ 11 files changed, 157 insertions(+), 32 deletions(-) create mode 100644 watcher/decision_engine/planner/manager.py create mode 100644 watcher/tests/decision_engine/planner/__init__.py rename watcher/tests/decision_engine/{ => planner}/test_default_planner.py (95%) create mode 100644 watcher/tests/decision_engine/planner/test_default_planner_loader.py create mode 100644 watcher/tests/decision_engine/planner/test_planner_manager.py diff --git a/etc/watcher/watcher.conf.sample b/etc/watcher/watcher.conf.sample index 4c00a4499..9c3f5be3b 100644 --- a/etc/watcher/watcher.conf.sample +++ b/etc/watcher/watcher.conf.sample @@ -91,7 +91,7 @@ # List of logger=LEVEL pairs. This option is ignored if # log_config_append is set. (list value) -#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN +#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN # Enables or disables publication of error events. (boolean value) #publish_errors = false @@ -127,10 +127,6 @@ # MatchMaker driver. (string value) #rpc_zmq_matchmaker = redis -# Use REQ/REP pattern for all methods CALL/CAST/FANOUT. (boolean -# value) -#rpc_zmq_all_req_rep = true - # Type of concurrency used. Either "native" or "eventlet" (string # value) #rpc_zmq_concurrency = eventlet @@ -157,8 +153,13 @@ # timeout exception when timeout expired. (integer value) #rpc_poll_timeout = 1 -# Configures zmq-messaging to use broker or not. (boolean value) -#zmq_use_broker = false +# Configures zmq-messaging to use proxy with non PUB/SUB patterns. +# (boolean value) +#direct_over_proxy = true + +# Use PUB/SUB pattern for fanout methods. PUB/SUB always uses proxy. +# (boolean value) +#use_pub_sub = true # Minimal port number for random ports range. (port value) # Minimum value: 1 @@ -633,17 +634,6 @@ # Deprecated group/name - [DEFAULT]/amqp_auto_delete #amqp_auto_delete = false -# Send a single AMQP reply to call message. The current behaviour -# since oslo-incubator is to send two AMQP replies - first one with -# the payload, a second one to ensure the other have finish to send -# the payload. We are going to remove it in the N release, but we must -# keep backward compatible at the same time. This option provides such -# compatibility - it defaults to False in Liberty and can be turned on -# for early adopters with a new installations or for testing. Please -# note, that this option will be removed in the Mitaka release. -# (boolean value) -#send_single_reply = false - # SSL version to use (valid only if SSL enabled). Valid values are # TLSv1 and SSLv23. SSLv2, SSLv3, TLSv1_1, and TLSv1_2 may be # available on some distributions. (string value) @@ -672,7 +662,7 @@ # replies. This value should not be longer than rpc_response_timeout. # (integer value) # Deprecated group/name - [DEFAULT]/kombu_reconnect_timeout -#kombu_missing_consumer_retry_timeout = 5 +#kombu_missing_consumer_retry_timeout = 60 # Determines how the next RabbitMQ node is chosen in case the one we # are currently connected to becomes unavailable. Takes effect only if @@ -803,3 +793,13 @@ # strategy (for example: BASIC_CONSOLIDATION:basic, # MY_GOAL:my_strategy_1) (dict value) #goals = DUMMY:dummy + + +[watcher_planner] + +# +# From watcher +# + +# The selected planner used to schedule the actions (string value) +#planner = default diff --git a/setup.cfg b/setup.cfg index 4d3d513c0..f6a194fc4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,6 +46,9 @@ watcher_strategies = basic = watcher.decision_engine.strategy.strategies.basic_consolidation:BasicConsolidation outlet_temp_control = watcher.decision_engine.strategy.strategies.outlet_temp_control:OutletTempControl +watcher_planners = + default = watcher.decision_engine.planner.default:DefaultPlanner + [build_sphinx] source-dir = doc/source build-dir = doc/build diff --git a/watcher/decision_engine/audit/default.py b/watcher/decision_engine/audit/default.py index 80f7a71f1..8f6221318 100644 --- a/watcher/decision_engine/audit/default.py +++ b/watcher/decision_engine/audit/default.py @@ -19,7 +19,7 @@ from watcher.common.messaging.events.event import Event from watcher.decision_engine.audit.base import \ BaseAuditHandler from watcher.decision_engine.messaging.events import Events -from watcher.decision_engine.planner.default import DefaultPlanner +from watcher.decision_engine.planner.manager import PlannerManager from watcher.decision_engine.strategy.context.default import \ DefaultStrategyContext from watcher.objects.audit import Audit @@ -34,6 +34,14 @@ class DefaultAuditHandler(BaseAuditHandler): super(DefaultAuditHandler, self).__init__() self._messaging = messaging self._strategy_context = DefaultStrategyContext() + self._planner_manager = PlannerManager() + self._planner = None + + @property + def planner(self): + if self._planner is None: + self._planner = self._planner_manager.load() + return self._planner @property def messaging(self): @@ -72,9 +80,7 @@ class DefaultAuditHandler(BaseAuditHandler): solution = self.strategy_context.execute_strategy(audit_uuid, request_context) - # schedule the actions and create in the watcher db the ActionPlan - planner = DefaultPlanner() - planner.schedule(request_context, audit.id, solution) + self.planner.schedule(request_context, audit.id, solution) # change state of the audit to SUCCEEDED self.update_audit_state(request_context, audit_uuid, diff --git a/watcher/decision_engine/planner/default.py b/watcher/decision_engine/planner/default.py index 80ea60ed9..76c838ba3 100644 --- a/watcher/decision_engine/planner/default.py +++ b/watcher/decision_engine/planner/default.py @@ -16,9 +16,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from oslo_log import log -from enum import Enum +import enum +from oslo_log import log from watcher._i18n import _LW from watcher.common import exception @@ -37,7 +37,7 @@ LOG = log.getLogger(__name__) # https://wiki.openstack.org/wiki/NovaOrchestration/WorkflowEngines​ -class Primitives(Enum): +class Primitives(enum.Enum): LIVE_MIGRATE = 'MIGRATE' COLD_MIGRATE = 'MIGRATE' POWER_STATE = 'POWERSTATE' diff --git a/watcher/decision_engine/planner/manager.py b/watcher/decision_engine/planner/manager.py new file mode 100644 index 000000000..b266718e1 --- /dev/null +++ b/watcher/decision_engine/planner/manager.py @@ -0,0 +1,53 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from oslo_config import cfg +from oslo_log import log + +from watcher.decision_engine.planner.loading import default as loader + + +LOG = log.getLogger(__name__) +CONF = cfg.CONF + +default_planner = 'default' + +WATCHER_PLANNER_OPTS = { + cfg.StrOpt('planner', + default=default_planner, + required=True, + help='The selected planner used to schedule the actions') +} +planner_opt_group = cfg.OptGroup(name='watcher_planner', + title='Defines the parameters of ' + 'the planner') +CONF.register_group(planner_opt_group) +CONF.register_opts(WATCHER_PLANNER_OPTS, planner_opt_group) + + +class PlannerManager(object): + def __init__(self): + self._loader = loader.DefaultPlannerLoader() + + @property + def loader(self): + return self._loader + + def load(self): + selected_planner = CONF.watcher_planner.planner + LOG.debug("Loading {0}".format(selected_planner)) + return self.loader.load(name=selected_planner) diff --git a/watcher/opts.py b/watcher/opts.py index a000fc204..d2d287047 100644 --- a/watcher/opts.py +++ b/watcher/opts.py @@ -18,6 +18,7 @@ import watcher.api.app from watcher.applier import manager as applier_manager from watcher.decision_engine import manager as decision_engine_manger +from watcher.decision_engine.planner import manager as planner_manager from watcher.decision_engine.strategy.selection import default \ as strategy_selector @@ -29,5 +30,6 @@ def list_opts(): ('watcher_decision_engine', decision_engine_manger.WATCHER_DECISION_ENGINE_OPTS), ('watcher_applier', - applier_manager.APPLIER_MANAGER_OPTS) + applier_manager.APPLIER_MANAGER_OPTS), + ('watcher_planner', planner_manager.WATCHER_PLANNER_OPTS) ] diff --git a/watcher/tests/decision_engine/planner/__init__.py b/watcher/tests/decision_engine/planner/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/tests/decision_engine/test_default_planner.py b/watcher/tests/decision_engine/planner/test_default_planner.py similarity index 95% rename from watcher/tests/decision_engine/test_default_planner.py rename to watcher/tests/decision_engine/planner/test_default_planner.py index 3f829e4ea..2b622f5b7 100644 --- a/watcher/tests/decision_engine/test_default_planner.py +++ b/watcher/tests/decision_engine/planner/test_default_planner.py @@ -15,9 +15,8 @@ # limitations under the License. import mock -from mock import MagicMock -from watcher.common.exception import ActionNotFound +from watcher.common import exception from watcher.common import utils from watcher.db import api as db_api from watcher.decision_engine.actions.base import BaseAction @@ -40,7 +39,7 @@ class SolutionFaker(object): metrics = FakerMetricsCollector() current_state_cluster = FakerModelCollector() sercon = BasicConsolidation("basic", "Basic offline consolidation") - sercon.ceilometer = MagicMock( + sercon.ceilometer = mock.MagicMock( get_statistics=metrics.mock_get_statistics) return sercon.execute(current_state_cluster.generate_scenario_1()) @@ -51,7 +50,7 @@ class SolutionFakerSingleHyp(object): metrics = FakerMetricsCollector() current_state_cluster = FakerModelCollector() sercon = BasicConsolidation("basic", "Basic offline consolidation") - sercon.ceilometer = MagicMock( + sercon.ceilometer = mock.MagicMock( get_statistics=metrics.mock_get_statistics) return sercon.execute( @@ -117,7 +116,8 @@ class TestDefaultPlanner(base.DbTestCase): audit = db_utils.create_test_audit(uuid=utils.generate_uuid()) fake_solution = SolutionFaker.build() fake_solution.actions[0] = "valeur_qcq" - self.assertRaises(ActionNotFound, self.default_planner.schedule, + self.assertRaises(exception.ActionNotFound, + self.default_planner.schedule, self.context, audit.id, fake_solution) def test_schedule_scheduled_empty(self): diff --git a/watcher/tests/decision_engine/planner/test_default_planner_loader.py b/watcher/tests/decision_engine/planner/test_default_planner_loader.py new file mode 100644 index 000000000..955fbdcbf --- /dev/null +++ b/watcher/tests/decision_engine/planner/test_default_planner_loader.py @@ -0,0 +1,29 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watcher.decision_engine.planner import base as planner +from watcher.decision_engine.planner.loading import default +from watcher.tests import base + + +class TestDefaultPlannerLoader(base.TestCase): + loader = default.DefaultPlannerLoader() + + def test_endpoints(self): + for endpoint in self.loader.list_available(): + loaded = self.loader.load(endpoint) + self.assertIsNotNone(loaded) + self.assertIsInstance(loaded, planner.BasePlanner) diff --git a/watcher/tests/decision_engine/planner/test_planner_manager.py b/watcher/tests/decision_engine/planner/test_planner_manager.py new file mode 100644 index 000000000..516502afb --- /dev/null +++ b/watcher/tests/decision_engine/planner/test_planner_manager.py @@ -0,0 +1,28 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +from watcher.decision_engine.planner.default import DefaultPlanner +from watcher.decision_engine.planner.manager import PlannerManager +from watcher.tests import base + + +class TestPlannerManager(base.TestCase): + def test_load(self): + cfg.CONF.set_override('planner', "default", group='watcher_planner') + manager = PlannerManager() + self.assertIsInstance(manager.load(), DefaultPlanner) diff --git a/watcher/tests/decision_engine/strategy/loading/test_default_strategy_loader.py b/watcher/tests/decision_engine/strategy/loading/test_default_strategy_loader.py index 44296cf3b..20170cc55 100644 --- a/watcher/tests/decision_engine/strategy/loading/test_default_strategy_loader.py +++ b/watcher/tests/decision_engine/strategy/loading/test_default_strategy_loader.py @@ -70,3 +70,7 @@ class TestDefaultStrategyLoader(TestCase): strategy_loader = DefaultStrategyLoader() loaded_strategy = strategy_loader.load("dummy") self.assertIsInstance(loaded_strategy, DummyStrategy) + + def test_endpoints(self): + for endpoint in self.strategy_loader.list_available(): + self.assertIsNotNone(self.strategy_loader.load(endpoint))