Merge "Adapt workload_stabilization strategy to new datasource backend"

This commit is contained in:
Zuul 2018-01-23 01:08:32 +00:00 committed by Gerrit Code Review
commit 40cff311c6
7 changed files with 34 additions and 139 deletions

View File

@ -108,8 +108,8 @@ class GnocchiHelper(base.DataSourceBase):
# measure has structure [time, granularity, value]
return statistics[-1][2]
def statistic_aggregation(self, resource_id, metric, period, granularity,
aggregation='mean'):
def statistic_aggregation(self, resource_id, metric, period, aggregation,
granularity=300):
stop_time = datetime.utcnow()
start_time = stop_time - timedelta(seconds=(int(period)))
return self._statistic_aggregation(

View File

@ -33,7 +33,7 @@ class DataSourceManager(object):
self._monasca = None
self._gnocchi = None
self.metric_map = base.DataSourceBase.METRIC_MAP
self.datasources = self.config.datasource
self.datasources = self.config.datasources
@property
def ceilometer(self):

View File

@ -28,7 +28,6 @@ It assumes that live migrations are possible in your cluster.
"""
import copy
import datetime
import itertools
import math
import random
@ -41,8 +40,6 @@ import oslo_utils
from watcher._i18n import _
from watcher.common import exception
from watcher.datasource import ceilometer as ceil
from watcher.datasource import gnocchi as gnoc
from watcher.decision_engine.model import element
from watcher.decision_engine.strategy.strategies import base
@ -73,9 +70,6 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
:param osc: :py:class:`~.OpenStackClients` instance
"""
super(WorkloadStabilization, self).__init__(config, osc)
self._ceilometer = None
self._gnocchi = None
self._nova = None
self.weights = None
self.metrics = None
self.thresholds = None
@ -169,43 +163,16 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
@classmethod
def get_config_opts(cls):
return [
cfg.StrOpt(
"datasource",
help="Data source to use in order to query the needed metrics",
default="gnocchi",
choices=["ceilometer", "gnocchi"])
cfg.ListOpt(
"datasources",
help="Datasources to use in order to query the needed metrics."
" If one of strategy metric isn't available in the first"
" datasource, the next datasource will be chosen.",
item_type=cfg.types.String(choices=['gnocchi', 'ceilometer',
'monasca']),
default=['gnocchi', 'ceilometer', 'monasca'])
]
@property
def ceilometer(self):
if self._ceilometer is None:
self.ceilometer = ceil.CeilometerHelper(osc=self.osc)
return self._ceilometer
@property
def nova(self):
if self._nova is None:
self.nova = self.osc.nova()
return self._nova
@nova.setter
def nova(self, n):
self._nova = n
@ceilometer.setter
def ceilometer(self, c):
self._ceilometer = c
@property
def gnocchi(self):
if self._gnocchi is None:
self.gnocchi = gnoc.GnocchiHelper(osc=self.osc)
return self._gnocchi
@gnocchi.setter
def gnocchi(self, gnocchi):
self._gnocchi = gnocchi
def transform_instance_cpu(self, instance_load, host_vcpus):
"""Transform instance cpu utilization to overall host cpu utilization.
@ -227,26 +194,9 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
LOG.debug('get_instance_load started')
instance_load = {'uuid': instance.uuid, 'vcpus': instance.vcpus}
for meter in self.metrics:
avg_meter = None
if self.config.datasource == "ceilometer":
avg_meter = self.ceilometer.statistic_aggregation(
resource_id=instance.uuid,
meter_name=meter,
period=self.periods['instance'],
aggregate='min'
)
elif self.config.datasource == "gnocchi":
stop_time = datetime.datetime.utcnow()
start_time = stop_time - datetime.timedelta(
seconds=int(self.periods['instance']))
avg_meter = self.gnocchi.statistic_aggregation(
resource_id=instance.uuid,
metric=meter,
granularity=self.granularity,
start_time=start_time,
stop_time=stop_time,
aggregation='mean'
)
avg_meter = self.datasource_backend.statistic_aggregation(
instance.uuid, meter, self.periods['instance'], 'mean',
granularity=self.granularity)
if avg_meter is None:
LOG.warning(
"No values returned by %(resource_id)s "
@ -287,25 +237,9 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
resource_id = "%s_%s" % (node.uuid, node.hostname)
else:
resource_id = node_id
if self.config.datasource == "ceilometer":
avg_meter = self.ceilometer.statistic_aggregation(
resource_id=resource_id,
meter_name=self.instance_metrics[metric],
period=self.periods['node'],
aggregate='avg'
)
elif self.config.datasource == "gnocchi":
stop_time = datetime.datetime.utcnow()
start_time = stop_time - datetime.timedelta(
seconds=int(self.periods['node']))
avg_meter = self.gnocchi.statistic_aggregation(
resource_id=resource_id,
metric=self.instance_metrics[metric],
granularity=self.granularity,
start_time=start_time,
stop_time=stop_time,
aggregation='mean'
)
avg_meter = self.datasource_backend.statistic_aggregation(
resource_id, self.instance_metrics[metric],
self.periods['node'], 'mean', granularity=self.granularity)
if avg_meter is None:
if meter_name == 'hardware.memory.used':

View File

@ -28,7 +28,7 @@ class TestDataSourceManager(base.BaseTestCase):
def test_get_backend(self, mock_gnoc):
manager = ds_manager.DataSourceManager(
config=mock.MagicMock(
datasource=['gnocchi', 'ceilometer', 'monasca']),
datasources=['gnocchi', 'ceilometer', 'monasca']),
osc=mock.MagicMock())
backend = manager.get_backend(['host_cpu_usage',
'instance_cpu_usage'])
@ -37,7 +37,7 @@ class TestDataSourceManager(base.BaseTestCase):
def test_get_backend_wrong_metric(self):
manager = ds_manager.DataSourceManager(
config=mock.MagicMock(
datasource=['gnocchi', 'ceilometer', 'monasca']),
datasources=['gnocchi', 'ceilometer', 'monasca']),
osc=mock.MagicMock())
self.assertRaises(exception.NoSuchMetric, manager.get_backend,
['host_cpu', 'instance_cpu_usage'])

View File

@ -26,6 +26,12 @@ class FakeCeilometerMetrics(object):
def empty_one_metric(self, emptytype):
self.emptytype = emptytype
# TODO(alexchadin): This method is added as temporary solution until
# all strategies use datasource_backend property.
def temp_mock_get_statistics(self, resource_id, meter_name, period,
aggregate, granularity=300):
return self.mock_get_statistics(resource_id, meter_name, period)
def mock_get_statistics(self, resource_id, meter_name, period,
aggregate='avg'):
result = 0

View File

@ -21,6 +21,13 @@ class FakeGnocchiMetrics(object):
def empty_one_metric(self, emptytype):
self.emptytype = emptytype
# TODO(alexchadin): This method is added as temporary solution until
# all strategies use datasource_backend property.
def temp_mock_get_statistics(self, resource_id, metric, period, aggregate,
granularity=300):
return self.mock_get_statistics(resource_id, metric, granularity,
0, 0, aggregation='mean')
def mock_get_statistics(self, resource_id, metric, granularity,
start_time, stop_time, aggregation='mean'):
result = 0

View File

@ -17,7 +17,6 @@
# limitations under the License.
#
import datetime
import mock
from watcher.common import clients
@ -69,7 +68,7 @@ class TestWorkloadStabilization(base.TestCase):
self.addCleanup(p_model.stop)
p_datasource = mock.patch.object(
strategies.WorkloadStabilization, self.datasource,
strategies.WorkloadStabilization, "datasource_backend",
new_callable=mock.PropertyMock)
self.m_datasource = p_datasource.start()
self.addCleanup(p_datasource.stop)
@ -84,7 +83,7 @@ class TestWorkloadStabilization(base.TestCase):
self.m_model.return_value = model_root.ModelRoot()
self.m_audit_scope.return_value = mock.Mock()
self.m_datasource.return_value = mock.Mock(
statistic_aggregation=self.fake_metrics.mock_get_statistics)
statistic_aggregation=self.fake_metrics.temp_mock_get_statistics)
self.strategy = strategies.WorkloadStabilization(
config=mock.Mock(datasource=self.datasource))
@ -121,57 +120,6 @@ class TestWorkloadStabilization(base.TestCase):
self.assertEqual(
instance_0_dict, self.strategy.get_instance_load(instance0))
def test_periods(self):
model = self.fake_cluster.generate_scenario_1()
self.m_model.return_value = model
p_ceilometer = mock.patch.object(
strategies.WorkloadStabilization, "ceilometer")
m_ceilometer = p_ceilometer.start()
self.addCleanup(p_ceilometer.stop)
p_gnocchi = mock.patch.object(strategies.WorkloadStabilization,
"gnocchi")
m_gnocchi = p_gnocchi.start()
self.addCleanup(p_gnocchi.stop)
datetime_patcher = mock.patch.object(
datetime, 'datetime',
mock.Mock(wraps=datetime.datetime)
)
mocked_datetime = datetime_patcher.start()
mocked_datetime.utcnow.return_value = datetime.datetime(
2017, 3, 19, 18, 53, 11, 657417)
self.addCleanup(datetime_patcher.stop)
m_ceilometer.return_value = mock.Mock(
statistic_aggregation=self.fake_metrics.mock_get_statistics)
m_gnocchi.return_value = mock.Mock(
statistic_aggregation=self.fake_metrics.mock_get_statistics)
instance0 = model.get_instance_by_uuid("INSTANCE_0")
self.strategy.get_instance_load(instance0)
if self.strategy.config.datasource == "ceilometer":
m_ceilometer.statistic_aggregation.assert_called_with(
aggregate='min', meter_name='memory.resident',
period=720, resource_id=instance0.uuid)
elif self.strategy.config.datasource == "gnocchi":
stop_time = datetime.datetime.utcnow()
start_time = stop_time - datetime.timedelta(
seconds=int('720'))
m_gnocchi.statistic_aggregation.assert_called_with(
resource_id=instance0.uuid, metric='memory.resident',
granularity=300, start_time=start_time, stop_time=stop_time,
aggregation='mean')
self.strategy.get_hosts_load()
if self.strategy.config.datasource == "ceilometer":
m_ceilometer.statistic_aggregation.assert_called_with(
aggregate='avg', meter_name='hardware.memory.used',
period=600, resource_id=mock.ANY)
elif self.strategy.config.datasource == "gnocchi":
stop_time = datetime.datetime.utcnow()
start_time = stop_time - datetime.timedelta(
seconds=int('600'))
m_gnocchi.statistic_aggregation.assert_called_with(
resource_id=mock.ANY, metric='hardware.memory.used',
granularity=300, start_time=start_time, stop_time=stop_time,
aggregation='mean')
def test_normalize_hosts_load(self):
self.m_model.return_value = self.fake_cluster.generate_scenario_1()
fake_hosts = {'Node_0': {'cpu_util': 0.07, 'memory.resident': 7},