openstack
/
aodh
Code Issues Proposed changes

Merge "Add Prometheus evaluator"

This commit is contained in:
Zuul 2023-11-07 16:02:59 +00:00 committed by Gerrit Code Review
parent 99df2a40bf f932265290
commit 197440c3c8
10 changed files with 226 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
aodh/api/controllers/v2/alarm_rules/composite.py
View File

@ -41,7 +41,8 @@ class CompositeRule(wtypes.UserType):
threshold_plugins = None
def __init__(self):
threshold_rules = ('gnocchi_resources_threshold',
threshold_rules = ('prometheus',
'gnocchi_resources_threshold',
'gnocchi_aggregation_by_metrics_threshold',
'gnocchi_aggregation_by_resources_threshold')
CompositeRule.threshold_plugins = named.NamedExtensionManager(

46
aodh/api/controllers/v2/alarm_rules/prometheus.py Normal file
View File

@ -0,0 +1,46 @@
#
# Copyright 2023 Red Hat, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log
import wsme
from wsme import types as wtypes
from aodh.api.controllers.v2 import base
LOG = log.getLogger(__name__)
class PrometheusRule(base.AlarmRule):
comparison_operator = base.AdvEnum('comparison_operator', str,
'lt', 'le', 'eq', 'ne', 'ge', 'gt',
default='eq')
"The comparison against the alarm threshold"
threshold = wsme.wsattr(float, mandatory=True)
"The threshold of the alarm"
query = wsme.wsattr(wtypes.text, mandatory=True)
"The Prometheus query"
@staticmethod
def validate(rule):
# TO-DO(mmagr): validate Prometheus query maybe?
return rule
def as_dict(self):
rule = self.as_dict_from_keys(['comparison_operator', 'threshold',
'query'])
return rule

3
aodh/evaluator/composite.py
View File

@ -116,7 +116,8 @@ class CompositeEvaluator(evaluator.Evaluator):
@property
def threshold_evaluators(self):
if not self._threshold_evaluators:
threshold_types = ('gnocchi_resources_threshold',
threshold_types = ('prometheus',
'gnocchi_resources_threshold',
'gnocchi_aggregation_by_metrics_threshold',
'gnocchi_aggregation_by_resources_threshold')
self._threshold_evaluators = stevedore.NamedExtensionManager(

78
aodh/evaluator/prometheus.py Normal file
View File

@ -0,0 +1,78 @@
#
# Copyright 2023 Red Hat, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from oslo_log import log
from observabilityclient import client
from aodh.evaluator import threshold
from aodh import keystone_client
LOG = log.getLogger(__name__)
OPTS = [
cfg.BoolOpt('prometheus_disable_rbac',
default=False,
help='Disable RBAC for Prometheus evaluator.'),
]
class PrometheusBase(threshold.ThresholdEvaluator):
def __init__(self, conf):
super(PrometheusBase, self).__init__(conf)
self._set_obsclient(conf)
self._no_rbac = conf.prometheus_disable_rbac
def _set_obsclient(self, conf):
session = keystone_client.get_session(conf)
opts = {'interface': conf.service_credentials.interface,
'region_name': conf.service_credentials.region_name}
self._prom = client.Client('1', session, adapter_options=opts)
def _get_metric_data(self, query):
LOG.debug(f'Querying Prometheus instance on: {query}')
return self._prom.query.query(query, disable_rbac=self._no_rbac)
class PrometheusEvaluator(PrometheusBase):
def _sanitize(self, metric_data):
sanitized = [float(m.value) for m in metric_data]
LOG.debug(f'Sanited Prometheus metric data: {metric_data}'
f' to statistics: {sanitized}')
return sanitized
def evaluate_rule(self, alarm_rule):
"""Evaluate alarm rule.
:returns: state, trending state, statistics, number of samples outside
threshold and reason
"""
metrics = self._get_metric_data(alarm_rule['query'])
if not metrics:
LOG.warning("Empty result fetched from Prometheus for query"
f" {alarm_rule['query']}")
statistics = self._sanitize(metrics)
if not statistics:
raise threshold.InsufficientDataError('datapoints are unknown',
statistics)
return self._process_statistics(alarm_rule, statistics)
def _unknown_reason_data(self, alarm, statistics):
LOG.warning(f'Transfering alarm {alarm} on unknown reason')
last = None if not statistics else statistics[-1]
return self._reason_data('unknown', len(statistics), last)

50
aodh/evaluator/threshold.py
View File

@ -96,19 +96,7 @@ class ThresholdEvaluator(evaluator.Evaluator):
' %(disposition)s threshold, most recent: %(most_recent)s'
% dict(reason_data, state=state), reason_data)
def evaluate_rule(self, alarm_rule):
"""Evaluate alarm rule.
:returns: state, trending state and statistics.
"""
start, end = self._bound_duration(alarm_rule)
statistics = self._statistics(alarm_rule, start, end)
statistics = self._sanitize(alarm_rule, statistics)
sufficient = len(statistics) >= alarm_rule['evaluation_periods']
if not sufficient:
raise InsufficientDataError(
'%d datapoints are unknown' % alarm_rule['evaluation_periods'],
statistics)
def _process_statistics(self, alarm_rule, statistics):
def _compare(value):
op = COMPARATORS[alarm_rule['comparison_operator']]
@ -129,6 +117,31 @@ class ThresholdEvaluator(evaluator.Evaluator):
trending_state = evaluator.ALARM if compared[-1] else evaluator.OK
return None, trending_state, statistics, number_outside, None
def evaluate_rule(self, alarm_rule):
"""Evaluate alarm rule.
:returns: state, trending state and statistics.
"""
start, end = self._bound_duration(alarm_rule)
statistics = self._statistics(alarm_rule, start, end)
statistics = self._sanitize(alarm_rule, statistics)
sufficient = len(statistics) >= alarm_rule['evaluation_periods']
if not sufficient:
raise InsufficientDataError(
'%d datapoints are unknown' % alarm_rule['evaluation_periods'],
statistics)
return self._process_statistics(alarm_rule, statistics)
def _unknown_reason_data(self, alarm, statistics):
LOG.warning(f'Expecting {alarm.rule["evaluation_periods"]} datapoints'
f' but only get {len(statistics)}')
# Reason is not same as log message because we want to keep
# consistent since thirdparty software may depend on old format.
last = None if not statistics else statistics[-1]
return self._reason_data('unknown', alarm.rule['evaluation_periods'],
last)
def _transition_alarm(self, alarm, state, trending_state, statistics,
outside_count, unknown_reason):
unknown = alarm.state == evaluator.UNKNOWN
@ -143,16 +156,7 @@ class ThresholdEvaluator(evaluator.Evaluator):
return
if state == evaluator.UNKNOWN and not unknown:
LOG.warning('Expecting %(expected)d datapoints but only get '
'%(actual)d'
% {'expected': alarm.rule['evaluation_periods'],
'actual': len(statistics)})
# Reason is not same as log message because we want to keep
# consistent since thirdparty software may depend on old format.
last = None if not statistics else statistics[-1]
reason_data = self._reason_data('unknown',
alarm.rule['evaluation_periods'],
last)
reason_data = self._unknown_reason_data(alarm, statistics)
self._refresh(alarm, state, unknown_reason, reason_data)
elif state and (alarm.state != state or continuous):

2
aodh/opts.py
View File

@ -23,6 +23,7 @@ import aodh.evaluator
import aodh.evaluator.event
import aodh.evaluator.gnocchi
import aodh.evaluator.loadbalancer
import aodh.evaluator.prometheus
import aodh.evaluator.threshold
import aodh.event
import aodh.keystone_client
@ -38,6 +39,7 @@ def list_opts():
itertools.chain(
aodh.evaluator.OPTS,
aodh.evaluator.event.OPTS,
aodh.evaluator.prometheus.OPTS,
aodh.evaluator.threshold.OPTS,
aodh.evaluator.loadbalancer.OPTS,
aodh.notifier.rest.OPTS,

7
aodh/tests/unit/evaluator/test_composite.py
View File

@ -15,6 +15,7 @@
from unittest import mock
import fixtures
import os
from oslo_utils import timeutils
from oslo_utils import uuidutils
@ -25,6 +26,12 @@ from aodh.tests import constants
from aodh.tests.unit.evaluator import base
# NOTE(mmagr): Overriding PrometheusEvaluator setting to avoid
# complains during init.
os.environ['PROMETHEUS_HOST'] = '127.0.0.1'
os.environ['PROMETHEUS_PORT'] = '666'
class BaseCompositeEvaluate(base.TestEvaluatorBase):
EVALUATOR = composite.CompositeEvaluator

59
aodh/tests/unit/test_evaluator.py
View File

@ -18,11 +18,14 @@ import fixtures
import time
from unittest import mock
from observabilityclient import prometheus_client
from oslo_config import fixture as fixture_config
from stevedore import extension
from aodh import evaluator
from aodh import service
from aodh.evaluator import prometheus
from aodh.tests import base as tests_base
@ -190,3 +193,59 @@ class TestAlarmEvaluationService(tests_base.BaseTestCase):
target = svc.partition_coordinator.extract_my_subset
self.assertEqual(0, target.call_count)
self.assertEqual(0, self.threshold_eval.evaluate.call_count)
class TestPrometheusEvaluator(tests_base.BaseTestCase):
def setUp(self):
super(TestPrometheusEvaluator, self).setUp()
conf = service.prepare_service(argv=[], config_files=[])
self.CONF = self.useFixture(fixture_config.Config(conf)).conf
def test_rule_evaluation(self):
metric_list = [
prometheus_client.PrometheusMetric({'metric': 'mtr',
'value': (0, 10)}),
prometheus_client.PrometheusMetric({'metric': 'mtr',
'value': (1, 15)}),
prometheus_client.PrometheusMetric({'metric': 'mtr',
'value': (2, 20)}),
prometheus_client.PrometheusMetric({'metric': 'mtr',
'value': (3, 25)}),
prometheus_client.PrometheusMetric({'metric': 'mtr',
'value': (4, 30)}),
prometheus_client.PrometheusMetric({'metric': 'mtr',
'value': (5, 15)}),
]
with mock.patch.object(prometheus.PrometheusEvaluator,
'_set_obsclient', return_value=None):
# mock Prometheus client
ev = prometheus.PrometheusEvaluator(self.CONF)
ev._get_metric_data = mock.Mock(return_value=metric_list)
# test transfer to alarm state
state, trend, stats, outside, reason = ev.evaluate_rule(
{'query': 'mtr', 'threshold': 9,
'comparison_operator': 'gt'})
self.assertEqual('alarm', state)
self.assertEqual(6, outside)
# test transfer to ok state
state, trend, stats, outside, reason = ev.evaluate_rule(
{'query': 'mtr', 'threshold': 31,
'comparison_operator': 'gt'})
self.assertEqual('ok', state)
self.assertEqual(0, outside)
# test trending to alarm state
state, trend, stats, outside, reason = ev.evaluate_rule(
{'query': 'mtr', 'threshold': 14,
'comparison_operator': 'gt'})
self.assertEqual('alarm', trend)
self.assertEqual(5, outside)
# test trending to ok state
state, trend, stats, outside, reason = ev.evaluate_rule(
{'query': 'mtr', 'threshold': 20,
'comparison_operator': 'gt'})
self.assertEqual('ok', trend)
self.assertEqual(2, outside)

1
requirements.txt
View File

@ -36,6 +36,7 @@ cachetools>=1.1.6
cotyledon>=1.7.3
keystoneauth1>=2.1
debtcollector>=1.2.0 # Apache-2.0
python-observabilityclient>=0.0.4
python-octaviaclient>=1.8.0
python-dateutil>=2.8.2 # BSD
python-heatclient>=1.17.0

2
setup.cfg
View File

@ -57,6 +57,7 @@ aodh.alarm.rule =
event = aodh.api.controllers.v2.alarm_rules.event:AlarmEventRule
composite = aodh.api.controllers.v2.alarm_rules.composite:composite_rule
loadbalancer_member_health = aodh.api.controllers.v2.alarm_rules.loadbalancer:LoadBalancerMemberHealthRule
prometheus = aodh.api.controllers.v2.alarm_rules.prometheus:PrometheusRule
aodh.evaluator =
gnocchi_resources_threshold = aodh.evaluator.gnocchi:GnocchiResourceThresholdEvaluator
@ -64,6 +65,7 @@ aodh.evaluator =
gnocchi_aggregation_by_resources_threshold = aodh.evaluator.gnocchi:GnocchiAggregationResourcesThresholdEvaluator
composite = aodh.evaluator.composite:CompositeEvaluator
loadbalancer_member_health = aodh.evaluator.loadbalancer:LoadBalancerMemberHealthEvaluator
prometheus = aodh.evaluator.prometheus:PrometheusEvaluator
aodh.notifier =
log = aodh.notifier.log:LogAlarmNotifier