Allow for global datasources preference from config

Allows to define a global preference for metric datasources with the
ability for strategy specific overrides. In addition, strategies which
do not require datasources have the config options removed this is
done to prevent confusion.

Some documentation that details the inner workings of selecting
datasources is updated.

Imports for some files in watcher/common have been changed to resolve
circular dependencies and now match the overall method to import
configuration.

Addtional datasources will be retrieved by the manager if the
datasource throws an error.

Implements: blueprint global-datasource-preference
Change-Id: I6fc455b288e338c20d2c4cfec5a0c95350bebc36

doc/source/contributor/plugin/strategy-plugin.rst

@@ -285,8 +285,15 @@ The following code snippet shows how datasource_backend is defined:
     @property
     def datasource_backend(self):
         if not self._datasource_backend:
+
+            # Load the global preferred datasources order but override it
+            # if the strategy has a specific datasources config
+            datasources = CONF.watcher_datasources
+            if self.config.datasources:
+                datasources = self.config
+
             self._datasource_backend = ds_manager.DataSourceManager(
-                config=self.config,
+                config=datasources,
                 osc=self.osc
             ).get_backend(self.DATASOURCE_METRICS)
         return self._datasource_backend

releasenotes/notes/global-datasource-preference-3ab47b4be09ff3a5.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    Watcher now supports configuring which datasource to use and in which
+    order. This configuration is done by specifying datasources in the
+    watcher_datasources section:
+
+    - ``[watcher_datasources] datasources = gnocchi,monasca,ceilometer``
+
+    Specific strategies can override this order and use datasources which
+    are not listed in the global preference.

watcher/common/clients.py

@@ -10,6 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from oslo_config import cfg
 
 from cinderclient import client as ciclient
 from glanceclient import client as glclient
@@ -23,15 +24,13 @@ from novaclient import client as nvclient
 
 from watcher.common import exception
 
-from watcher import conf
-
 try:
     from ceilometerclient import client as ceclient
     HAS_CEILCLIENT = True
 except ImportError:
     HAS_CEILCLIENT = False
 
-CONF = conf.CONF
+CONF = cfg.CONF
 
 _CLIENTS_AUTH_GROUP = 'watcher_clients_auth'
 

watcher/common/exception.py

@@ -26,16 +26,15 @@ import functools
 import sys
 
 from keystoneclient import exceptions as keystone_exceptions
+from oslo_config import cfg
 from oslo_log import log
 import six
 
 from watcher._i18n import _
 
-from watcher import conf
-
 LOG = log.getLogger(__name__)
 
-CONF = conf.CONF
+CONF = cfg.CONF
 
 
 def wrap_keystone_exception(func):

watcher/common/utils.py

@@ -24,6 +24,7 @@ import string
 from croniter import croniter
 
 from jsonschema import validators
+from oslo_config import cfg
 from oslo_log import log
 from oslo_utils import strutils
 from oslo_utils import uuidutils
@@ -31,9 +32,7 @@ import six
 
 from watcher.common import exception
 
-from watcher import conf
-
-CONF = conf.CONF
+CONF = cfg.CONF
 
 LOG = log.getLogger(__name__)
 

watcher/conf/__init__.py

@@ -25,6 +25,7 @@ from watcher.conf import ceilometer_client
 from watcher.conf import cinder_client
 from watcher.conf import clients_auth
 from watcher.conf import collector
+from watcher.conf import datasources
 from watcher.conf import db
 from watcher.conf import decision_engine
 from watcher.conf import exception
@@ -44,6 +45,7 @@ service.register_opts(CONF)
 api.register_opts(CONF)
 paths.register_opts(CONF)
 exception.register_opts(CONF)
+datasources.register_opts(CONF)
 db.register_opts(CONF)
 planner.register_opts(CONF)
 applier.register_opts(CONF)

watcher/conf/datasources.py

@@ -0,0 +1,47 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2019 European Organization for Nuclear Research (CERN)
+#
+# Authors: Corne Lukken <info@dantalion.nl>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+
+from watcher.datasources import manager
+
+datasources = cfg.OptGroup(name='watcher_datasources',
+                           title='Configuration Options for watcher'
+                                 ' datasources')
+
+possible_datasources = list(manager.DataSourceManager.metric_map.keys())
+
+DATASOURCES_OPTS = [
+    cfg.ListOpt("datasources",
+                help="Datasources to use in order to query the needed metrics."
+                     " If one of strategy metric is not available in the first"
+                     " datasource, the next datasource will be chosen. This is"
+                     " the default for all strategies unless a strategy has a"
+                     " specific override.",
+                item_type=cfg.types.String(choices=possible_datasources),
+                default=possible_datasources)
+    ]
+
+
+def register_opts(conf):
+    conf.register_group(datasources)
+    conf.register_opts(DATASOURCES_OPTS, group=datasources)
+
+
+def list_opts():
+    return [('watcher_datasources', DATASOURCES_OPTS)]

watcher/datasources/ceilometer.py

@@ -25,7 +25,7 @@ from oslo_utils import timeutils
 from watcher._i18n import _
 from watcher.common import clients
 from watcher.common import exception
-from watcher.datasource import base
+from watcher.datasources import base
 
 
 LOG = log.getLogger(__name__)

watcher/datasources/gnocchi.py

@@ -26,7 +26,7 @@ from oslo_log import log
 from watcher.common import clients
 from watcher.common import exception
 from watcher.common import utils as common_utils
-from watcher.datasource import base
+from watcher.datasources import base
 
 CONF = cfg.CONF
 LOG = log.getLogger(__name__)

watcher/datasources/manager.py

@@ -13,25 +13,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections import OrderedDict
+
 from watcher.common import exception
-from watcher.datasource import ceilometer as ceil
-from watcher.datasource import gnocchi as gnoc
-from watcher.datasource import monasca as mon
+from watcher.datasources import ceilometer as ceil
+from watcher.datasources import gnocchi as gnoc
+from watcher.datasources import monasca as mon
 
 
 class DataSourceManager(object):
 
+    metric_map = OrderedDict([
+        (gnoc.GnocchiHelper.NAME, gnoc.GnocchiHelper.METRIC_MAP),
+        (ceil.CeilometerHelper.NAME, ceil.CeilometerHelper.METRIC_MAP),
+        (mon.MonascaHelper.NAME, mon.MonascaHelper.METRIC_MAP),
+    ])
+    """Dictionary with all possible datasources, dictionary order is the default
+    order for attempting to use datasources
+    """
+
     def __init__(self, config=None, osc=None):
         self.osc = osc
         self.config = config
         self._ceilometer = None
         self._monasca = None
         self._gnocchi = None
-        self.metric_map = {
-            mon.MonascaHelper.NAME: mon.MonascaHelper.METRIC_MAP,
-            gnoc.GnocchiHelper.NAME: gnoc.GnocchiHelper.METRIC_MAP,
-            ceil.CeilometerHelper.NAME: ceil.CeilometerHelper.METRIC_MAP
-        }
         self.datasources = self.config.datasources
 
     @property
@@ -73,5 +79,10 @@ class DataSourceManager(object):
                         no_metric = True
                         break
             if not no_metric:
-                return getattr(self, datasource)
+                # Try to use a specific datasource but attempt additional
+                # datasources upon exceptions (if config has more datasources)
+                try:
+                    return getattr(self, datasource)
+                except Exception:
+                    pass
         raise exception.NoSuchMetric()

watcher/datasources/monasca.py

@@ -22,7 +22,7 @@ from monascaclient import exc
 
 from watcher.common import clients
 from watcher.common import exception
-from watcher.datasource import base
+from watcher.datasources import base
 
 
 class MonascaHelper(base.DataSourceBase):

watcher/decision_engine/strategy/strategies/actuation.py

@@ -80,6 +80,12 @@ class Actuator(base.UnclassifiedStrategy):
             ]
         }
 
+    @classmethod
+    def get_config_opts(cls):
+        """Override base class config options as do not use datasource """
+
+        return []
+
     @property
     def actions(self):
         return self.input_parameters.get('actions', [])

watcher/decision_engine/strategy/strategies/base.py

@@ -48,7 +48,7 @@ from watcher.common import context
 from watcher.common import exception
 from watcher.common.loader import loadable
 from watcher.common import utils
-from watcher.datasource import manager as ds_manager
+from watcher.datasources import manager as ds_manager
 from watcher.decision_engine.loading import default as loading
 from watcher.decision_engine.model.collector import manager
 from watcher.decision_engine.solution import default
@@ -130,6 +130,8 @@ class BaseStrategy(loadable.Loadable):
     """
 
     DATASOURCE_METRICS = []
+    """Contains all metrics the strategy requires from a datasource to properly
+    execute"""
 
     def __init__(self, config, osc=None):
         """Constructor: the signature should be identical within the subclasses
@@ -197,7 +199,18 @@ class BaseStrategy(loadable.Loadable):
         :return: A list of configuration options relative to this Loadable
         :rtype: list of :class:`oslo_config.cfg.Opt` instances
         """
-        return []
+
+        datasources_ops = list(ds_manager.DataSourceManager.metric_map.keys())
+
+        return [
+            cfg.ListOpt(
+                "datasources",
+                help="Datasources to use in order to query the needed metrics."
+                     " This option overrides the global preference."
+                     " options: {0}".format(datasources_ops),
+                item_type=cfg.types.String(choices=datasources_ops),
+                default=None)
+        ]
 
     @abc.abstractmethod
     def pre_execute(self):
@@ -341,8 +354,15 @@ class BaseStrategy(loadable.Loadable):
     @property
     def datasource_backend(self):
         if not self._datasource_backend:
+
+            # Load the global preferred datasources order but override it
+            # if the strategy has a specific datasources config
+            datasources = CONF.watcher_datasources
+            if self.config.datasources:
+                datasources = self.config
+
             self._datasource_backend = ds_manager.DataSourceManager(
-                config=self.config,
+                config=datasources,
                 osc=self.osc
             ).get_backend(self.DATASOURCE_METRICS)
         return self._datasource_backend
@@ -429,6 +449,12 @@ class DummyBaseStrategy(BaseStrategy):
     def get_goal_name(cls):
         return "dummy"
 
+    @classmethod
+    def get_config_opts(cls):
+        """Override base class config options as do not use datasource """
+
+        return []
+
 
 @six.add_metaclass(abc.ABCMeta)
 class UnclassifiedStrategy(BaseStrategy):
@@ -486,6 +512,12 @@ class SavingEnergyBaseStrategy(BaseStrategy):
     def get_goal_name(cls):
         return "saving_energy"
 
+    @classmethod
+    def get_config_opts(cls):
+        """Override base class config options as do not use datasource """
+
+        return []
+
 
 @six.add_metaclass(abc.ABCMeta)
 class ZoneMigrationBaseStrategy(BaseStrategy):
@@ -494,6 +526,12 @@ class ZoneMigrationBaseStrategy(BaseStrategy):
     def get_goal_name(cls):
         return "hardware_maintenance"
 
+    @classmethod
+    def get_config_opts(cls):
+        """Override base class config options as do not use datasource """
+
+        return []
+
 
 @six.add_metaclass(abc.ABCMeta)
 class HostMaintenanceBaseStrategy(BaseStrategy):
@@ -503,3 +541,9 @@ class HostMaintenanceBaseStrategy(BaseStrategy):
     @classmethod
     def get_goal_name(cls):
         return "cluster_maintaining"
+
+    @classmethod
+    def get_config_opts(cls):
+        """Override base class config options as do not use datasource """
+
+        return []

watcher/decision_engine/strategy/strategies/basic_consolidation.py

@@ -172,19 +172,11 @@ class BasicConsolidation(base.ServerConsolidationBaseStrategy):
 
     @classmethod
     def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca']),
+        return super(BasicConsolidation, cls).get_config_opts() + [
             cfg.BoolOpt(
-                "check_optimize_metadata",
-                help="Check optimize metadata field in instance before "
-                     "migration",
+                'check_optimize_metadata',
+                help='Check optimize metadata field in instance before'
+                     ' migration',
                 default=False),
         ]
 

watcher/decision_engine/strategy/strategies/noisy_neighbor.py

@@ -94,19 +94,6 @@ class NoisyNeighbor(base.NoisyNeighborBaseStrategy):
             },
         }
 
-    @classmethod
-    def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca'])
-        ]
-
     def get_current_and_previous_cache(self, instance):
         try:
             curr_cache = self.datasource_backend.get_instance_l3_cache_usage(

watcher/decision_engine/strategy/strategies/outlet_temp_control.py

@@ -32,7 +32,6 @@ thermal condition (lowest outlet temperature) when the outlet temperature
 of source hosts reach a configurable threshold.
 """
 
-from oslo_config import cfg
 from oslo_log import log
 
 from watcher._i18n import _
@@ -141,19 +140,6 @@ class OutletTempControl(base.ThermalOptimizationBaseStrategy):
     def granularity(self):
         return self.input_parameters.get('granularity', 300)
 
-    @classmethod
-    def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca']),
-        ]
-
     def get_available_compute_nodes(self):
         default_node_scope = [element.ServiceState.ENABLED.value]
         return {uuid: cn for uuid, cn in

watcher/decision_engine/strategy/strategies/storage_capacity_balance.py

@@ -99,7 +99,7 @@ class StorageCapacityBalance(base.WorkloadStabilizationBaseStrategy):
 
     @classmethod
     def get_config_opts(cls):
-        return [
+        return super(StorageCapacityBalance, cls).get_config_opts() + [
             cfg.ListOpt(
                 "ex_pools",
                 help="exclude pools",

watcher/decision_engine/strategy/strategies/uniform_airflow.py

@@ -17,7 +17,6 @@
 # limitations under the License.
 #
 
-from oslo_config import cfg
 from oslo_log import log
 
 from watcher._i18n import _
@@ -148,19 +147,6 @@ class UniformAirflow(base.BaseStrategy):
             },
         }
 
-    @classmethod
-    def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca']),
-        ]
-
     def get_available_compute_nodes(self):
         default_node_scope = [element.ServiceState.ENABLED.value]
         return {uuid: cn for uuid, cn in

watcher/decision_engine/strategy/strategies/vm_workload_consolidation.py

@@ -18,7 +18,6 @@
 # limitations under the License.
 #
 
-from oslo_config import cfg
 from oslo_log import log
 import six
 
@@ -138,19 +137,6 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy):
             }
         }
 
-    @classmethod
-    def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca'])
-        ]
-
     def get_available_compute_nodes(self):
         default_node_scope = [element.ServiceState.ENABLED.value,
                               element.ServiceState.DISABLED.value]

watcher/decision_engine/strategy/strategies/workload_balance.py

@@ -19,7 +19,6 @@
 
 from __future__ import division
 
-from oslo_config import cfg
 from oslo_log import log
 
 from watcher._i18n import _
@@ -130,19 +129,6 @@ class WorkloadBalance(base.WorkloadStabilizationBaseStrategy):
             },
         }
 
-    @classmethod
-    def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca'])
-        ]
-
     def get_available_compute_nodes(self):
         default_node_scope = [element.ServiceState.ENABLED.value]
         return {uuid: cn for uuid, cn in

watcher/decision_engine/strategy/strategies/workload_stabilization.py

@@ -227,19 +227,6 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
             }
         }
 
-    @classmethod
-    def get_config_opts(cls):
-        return [
-            cfg.ListOpt(
-                "datasources",
-                help="Datasources to use in order to query the needed metrics."
-                     " If one of strategy metric isn't available in the first"
-                     " datasource, the next datasource will be chosen.",
-                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
-                                                    'monasca']),
-                default=['gnocchi', 'ceilometer', 'monasca'])
-        ]
-
     def transform_instance_cpu(self, instance_load, host_vcpus):
         """Transform instance cpu utilization to overall host cpu utilization.
 

watcher/tests/conf/test_list_opts.py

@@ -29,8 +29,8 @@ class TestListOpts(base.TestCase):
         super(TestListOpts, self).setUp()
         self.base_sections = [
             'DEFAULT', 'api', 'database', 'watcher_decision_engine',
-            'watcher_applier', 'watcher_planner', 'nova_client',
-            'glance_client', 'gnocchi_client', 'cinder_client',
+            'watcher_applier', 'watcher_datasources', 'watcher_planner',
+            'nova_client', 'glance_client', 'gnocchi_client', 'cinder_client',
             'ceilometer_client', 'monasca_client', 'ironic_client',
             'neutron_client', 'watcher_clients_auth', 'collector']
         self.opt_sections = list(dict(opts.list_opts()).keys())

watcher/tests/datasources/test_ceilometer_helper.py

@@ -20,7 +20,7 @@ from __future__ import unicode_literals
 import mock
 
 from watcher.common import clients
-from watcher.datasource import ceilometer as ceilometer_helper
+from watcher.datasources import ceilometer as ceilometer_helper
 from watcher.tests import base
 
 

watcher/tests/datasources/test_gnocchi_helper.py

@@ -18,7 +18,7 @@ import mock
 from oslo_config import cfg
 
 from watcher.common import clients
-from watcher.datasource import gnocchi as gnocchi_helper
+from watcher.datasources import gnocchi as gnocchi_helper
 from watcher.tests import base
 
 CONF = cfg.CONF

watcher/tests/datasources/test_manager.py

@@ -17,7 +17,8 @@
 import mock
 
 from watcher.common import exception
-from watcher.datasource import manager as ds_manager
+from watcher.datasources import gnocchi
+from watcher.datasources import manager as ds_manager
 from watcher.tests import base
 
 
@@ -46,3 +47,13 @@ class TestDataSourceManager(base.BaseTestCase):
             osc=mock.MagicMock())
         self.assertRaises(exception.NoSuchMetric, manager.get_backend,
                           ['host_cpu', 'instance_cpu_usage'])
+
+    @mock.patch.object(gnocchi, 'GnocchiHelper')
+    def test_get_backend_error_datasource(self, m_gnocchi):
+        m_gnocchi.side_effect = exception.DataSourceNotAvailable
+        manager = ds_manager.DataSourceManager(
+            config=mock.MagicMock(
+                datasources=['gnocchi', 'ceilometer', 'monasca']),
+            osc=mock.MagicMock())
+        backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage'])
+        self.assertEqual(backend, manager.ceilometer)

watcher/tests/datasources/test_monasca_helper.py

@@ -18,7 +18,7 @@ import mock
 from oslo_config import cfg
 
 from watcher.common import clients
-from watcher.datasource import monasca as monasca_helper
+from watcher.datasources import monasca as monasca_helper
 from watcher.tests import base
 
 CONF = cfg.CONF

watcher/tests/decision_engine/strategy/strategies/test_base.py

@@ -17,6 +17,7 @@
 import mock
 
 from watcher.common import exception
+from watcher.datasources import manager
 from watcher.decision_engine.model import model_root
 from watcher.decision_engine.strategy import strategies
 from watcher.tests import base
@@ -49,6 +50,67 @@ class TestBaseStrategy(base.TestCase):
         self.strategy = strategies.DummyStrategy(config=mock.Mock())
 
 
+class TestBaseStrategyDatasource(TestBaseStrategy):
+
+    def setUp(self):
+        super(TestBaseStrategyDatasource, self).setUp()
+        self.strategy = strategies.DummyStrategy(
+            config=mock.Mock(datasources=None))
+
+    @mock.patch.object(strategies.BaseStrategy, 'osc', None)
+    @mock.patch.object(manager, 'DataSourceManager')
+    @mock.patch.object(strategies.base, 'CONF')
+    def test_global_preference(self, m_conf, m_manager):
+        """Test if the global preference is used"""
+
+        m_conf.watcher_datasources.datasources = \
+            ['gnocchi', 'monasca', 'ceilometer']
+
+        # Access the property so that the configuration is read in order to
+        # get the correct datasource
+        self.strategy.datasource_backend()
+
+        m_manager.assert_called_once_with(
+            config=m_conf.watcher_datasources, osc=None)
+
+    @mock.patch.object(strategies.BaseStrategy, 'osc', None)
+    @mock.patch.object(manager, 'DataSourceManager')
+    @mock.patch.object(strategies.base, 'CONF')
+    def test_global_preference_reverse(self, m_conf, m_manager):
+        """Test if the global preference is used with another order"""
+
+        m_conf.watcher_datasources.datasources = \
+            ['ceilometer', 'monasca', 'gnocchi']
+
+        # Access the property so that the configuration is read in order to
+        # get the correct datasource
+        self.strategy.datasource_backend()
+
+        m_manager.assert_called_once_with(
+            config=m_conf.watcher_datasources, osc=None)
+
+    @mock.patch.object(strategies.BaseStrategy, 'osc', None)
+    @mock.patch.object(manager, 'DataSourceManager')
+    @mock.patch.object(strategies.base, 'CONF')
+    def test_strategy_preference_override(self, m_conf, m_manager):
+        """Test if the global preference can be overridden"""
+
+        datasources = mock.Mock(datasources=['ceilometer'])
+
+        self.strategy = strategies.DummyStrategy(
+            config=datasources)
+
+        m_conf.watcher_datasources.datasources = \
+            ['ceilometer', 'monasca', 'gnocchi']
+
+        # Access the property so that the configuration is read in order to
+        # get the correct datasource
+        self.strategy.datasource_backend()
+
+        m_manager.assert_called_once_with(
+            config=datasources, osc=None)
+
+
 class TestBaseStrategyException(TestBaseStrategy):
 
     def setUp(self):