Adapt noisy_neighbor strategy to multiple datasource backend

Partially-Implements: blueprint watcher-multi-datasource
Change-Id: Ibcd5d0776280bb68ed838f88ebfcde27fc1a3d35

watcher/datasource/base.py

@@ -33,7 +33,7 @@ class DataSourceBase(object):
                         host_memory_usage='hardware.memory.used', ),
         gnocchi=dict(host_cpu_usage='compute.node.cpu.percent',
                      instance_cpu_usage='cpu_util',
-                     instance_l3_cache_usage=None,
+                     instance_l3_cache_usage='cpu_l3_cache',
                      host_outlet_temp='hardware.ipmi.node.outlet_temperature',
                      host_airflow='hardware.ipmi.node.airflow',
                      host_inlet_temp='hardware.ipmi.node.temperature',

watcher/datasource/gnocchi.py

@@ -146,7 +146,9 @@ class GnocchiHelper(base.DataSourceBase):
 
     def get_instance_l3_cache_usage(self, resource_id, period, aggregate,
                                     granularity=300):
-        raise NotImplementedError
+        meter_name = self.METRIC_MAP.get('instance_l3_cache_usage')
+        return self.statistic_aggregation(resource_id, meter_name, period,
+                                          granularity, aggregation=aggregate)
 
     def get_instance_ram_allocated(self, resource_id, period, aggregate,
                                    granularity=300):

watcher/decision_engine/strategy/strategies/noisy_neighbor.py

@@ -16,14 +16,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+from oslo_config import cfg
 from oslo_log import log
 
 from watcher._i18n import _
 from watcher.common import exception as wexc
-from watcher.datasource import ceilometer as ceil
 from watcher.decision_engine.strategy.strategies import base
 
 LOG = log.getLogger(__name__)
+CONF = cfg.CONF
 
 
 class NoisyNeighbor(base.NoisyNeighborBaseStrategy):
@@ -45,17 +46,6 @@ class NoisyNeighbor(base.NoisyNeighborBaseStrategy):
         super(NoisyNeighbor, self).__init__(config, osc)
 
         self.meter_name = self.METER_NAME_L3
-        self._ceilometer = None
-
-    @property
-    def ceilometer(self):
-        if self._ceilometer is None:
-            self.ceilometer = ceil.CeilometerHelper(osc=self.osc)
-        return self._ceilometer
-
-    @ceilometer.setter
-    def ceilometer(self, c):
-        self._ceilometer = c
 
     @classmethod
     def get_name(cls):
@@ -81,32 +71,39 @@ class NoisyNeighbor(base.NoisyNeighborBaseStrategy):
                     "default": 35.0
                 },
                 "period": {
-                    "description": "Aggregate time period of ceilometer",
+                    "description": "Aggregate time period of "
+                                   "ceilometer and gnocchi",
                     "type": "number",
                     "default": 100.0
                 },
             },
         }
 
+    @classmethod
+    def get_config_opts(cls):
+        return [
+            cfg.ListOpt(
+                "datasource",
+                help="Data source to use in order to query the needed metrics",
+                item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
+                                                    'monasca']),
+                default=['gnocchi', 'ceilometer', 'monasca'])
+        ]
+
     def get_current_and_previous_cache(self, instance):
-
         try:
-            current_cache = self.ceilometer.statistic_aggregation(
+            curr_cache = self.datasource_backend.get_instance_l3_cache_usage(
-                resource_id=instance.uuid,
+                instance.uuid, self.period, 'mean', granularity=300)
-                meter_name=self.meter_name, period=self.period,
-                aggregate='avg')
-
             previous_cache = 2 * (
-                self.ceilometer.statistic_aggregation(
+                self.datasource_backend.get_instance_l3_cache_usage(
-                    resource_id=instance.uuid,
+                    instance.uuid, 2 * self.period,
-                    meter_name=self.meter_name,
+                    'mean', granularity=300)) - curr_cache
-                    period=2*self.period, aggregate='avg')) - current_cache
 
         except Exception as exc:
             LOG.exception(exc)
-            return None
+            return None, None
 
-        return current_cache, previous_cache
+        return curr_cache, previous_cache
 
     def find_priority_instance(self, instance):
 
@@ -114,7 +111,7 @@ class NoisyNeighbor(base.NoisyNeighborBaseStrategy):
             self.get_current_and_previous_cache(instance)
 
         if None in (current_cache, previous_cache):
-            LOG.warning("Ceilometer unable to pick L3 Cache "
+            LOG.warning("Datasource unable to pick L3 Cache "
                         "values. Skipping the instance")
             return None
 
@@ -130,7 +127,7 @@ class NoisyNeighbor(base.NoisyNeighborBaseStrategy):
             self.get_current_and_previous_cache(instance)
 
         if None in (noisy_current_cache, noisy_previous_cache):
-            LOG.warning("Ceilometer unable to pick "
+            LOG.warning("Datasource unable to pick "
                         "L3 Cache. Skipping the instance")
             return None
 

watcher/tests/decision_engine/model/ceilometer_metrics.py

@@ -64,12 +64,12 @@ class FakeCeilometerMetrics(object):
             result = self.get_average_usage_instance_memory_wb(resource_id)
         return result
 
-    def mock_get_statistics_nn(self, resource_id, meter_name, period,
+    def mock_get_statistics_nn(self, resource_id, period,
-                               aggregate='avg'):
+                               aggregation, granularity=300):
         result = 0.0
-        if meter_name == "cpu_l3_cache" and period == 100:
+        if period == 100:
             result = self.get_average_l3_cache_current(resource_id)
-        if meter_name == "cpu_l3_cache" and period == 200:
+        if period == 200:
             result = self.get_average_l3_cache_previous(resource_id)
         return result
 

watcher/tests/decision_engine/model/gnocchi_metrics.py

@@ -52,6 +52,41 @@ class FakeGnocchiMetrics(object):
             result = self.get_average_power(resource_id)
         return result
 
+    def mock_get_statistics_nn(self, resource_id, period,
+                               aggregation, granularity=300):
+        result = 0.0
+        if period == 100:
+            result = self.get_average_l3_cache_current(resource_id)
+        if period == 200:
+            result = self.get_average_l3_cache_previous(resource_id)
+        return result
+
+    @staticmethod
+    def get_average_l3_cache_current(uuid):
+        """The average l3 cache used by instance"""
+        mock = {}
+        mock['73b09e16-35b7-4922-804e-e8f5d9b740fc'] = 35 * oslo_utils.units.Ki
+        mock['cae81432-1631-4d4e-b29c-6f3acdcde906'] = 30 * oslo_utils.units.Ki
+        mock['INSTANCE_3'] = 40 * oslo_utils.units.Ki
+        mock['INSTANCE_4'] = 35 * oslo_utils.units.Ki
+        if uuid not in mock.keys():
+            mock[uuid] = 25 * oslo_utils.units.Ki
+        return mock[str(uuid)]
+
+    @staticmethod
+    def get_average_l3_cache_previous(uuid):
+        """The average l3 cache used by instance"""
+        mock = {}
+        mock['73b09e16-35b7-4922-804e-e8f5d9b740fc'] = 34.5 * (
+            oslo_utils.units.Ki)
+        mock['cae81432-1631-4d4e-b29c-6f3acdcde906'] = 30.5 * (
+            oslo_utils.units.Ki)
+        mock['INSTANCE_3'] = 60 * oslo_utils.units.Ki
+        mock['INSTANCE_4'] = 22.5 * oslo_utils.units.Ki
+        if uuid not in mock.keys():
+            mock[uuid] = 25 * oslo_utils.units.Ki
+        return mock[str(uuid)]
+
     def mock_get_statistics_wb(self, resource_id, metric, granularity,
                                start_time, stop_time, aggregation='mean'):
         result = 0.0

watcher/tests/decision_engine/strategy/strategies/test_noisy_neighbor.py

@@ -27,14 +27,24 @@ from watcher.decision_engine.strategy import strategies
 from watcher.tests import base
 from watcher.tests.decision_engine.model import ceilometer_metrics
 from watcher.tests.decision_engine.model import faker_cluster_state
+from watcher.tests.decision_engine.model import gnocchi_metrics
 
 
 class TestNoisyNeighbor(base.TestCase):
 
+    scenarios = [
+        ("Ceilometer",
+         {"datasource": "ceilometer",
+          "fake_datasource_cls": ceilometer_metrics.FakeCeilometerMetrics}),
+        ("Gnocchi",
+         {"datasource": "gnocchi",
+          "fake_datasource_cls": gnocchi_metrics.FakeGnocchiMetrics}),
+    ]
+
     def setUp(self):
         super(TestNoisyNeighbor, self).setUp()
         # fake metrics
-        self.fake_metrics = ceilometer_metrics.FakeCeilometerMetrics()
+        self.f_metrics = self.fake_datasource_cls()
         # fake cluster
         self.fake_cluster = faker_cluster_state.FakerModelCollector()
 
@@ -44,11 +54,11 @@ class TestNoisyNeighbor(base.TestCase):
         self.m_model = p_model.start()
         self.addCleanup(p_model.stop)
 
-        p_ceilometer = mock.patch.object(
+        p_datasource = mock.patch.object(
-            strategies.NoisyNeighbor, "ceilometer",
+            strategies.NoisyNeighbor, "datasource_backend",
             new_callable=mock.PropertyMock)
-        self.m_ceilometer = p_ceilometer.start()
+        self.m_datasource = p_datasource.start()
-        self.addCleanup(p_ceilometer.stop)
+        self.addCleanup(p_datasource.stop)
 
         p_audit_scope = mock.patch.object(
             strategies.NoisyNeighbor, "audit_scope",
@@ -60,8 +70,8 @@ class TestNoisyNeighbor(base.TestCase):
         self.m_audit_scope.return_value = mock.Mock()
 
         self.m_model.return_value = model_root.ModelRoot()
-        self.m_ceilometer.return_value = mock.Mock(
+        self.m_datasource.return_value = mock.Mock(
-            statistic_aggregation=self.fake_metrics.mock_get_statistics_nn)
+            get_instance_l3_cache_usage=self.f_metrics.mock_get_statistics_nn)
         self.strategy = strategies.NoisyNeighbor(config=mock.Mock())
 
         self.strategy.input_parameters = utils.Struct()