openstack
/
vitrage
Code Issues Proposed changes

Adjust Vitrage api to support Prometheus datasource 

This includes adding support to basic mode auth along side to keystone auth.

documentation commits coming after

Change-Id: If99fd31dae55b30f199f261adb6a6de933857ad2
This commit is contained in:
Muhamad Najjar 2018-07-22 15:46:53 +00:00
parent 98d6401050
commit 034a597c9f
9 changed files with 454 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
devstack/apache-vitrage.template
View File

@ -4,6 +4,7 @@ Listen %PORT%
WSGIDaemonProcess vitrage-api processes=%APIWORKERS% threads=10 user=%USER% display-name=%{GROUP} %VIRTUALENV%
WSGIProcessGroup vitrage-api
WSGIScriptAlias / %WSGIAPP%
WSGIPassAuthorization On
WSGIApplicationGroup %{GLOBAL}
<IfVersion >= 2.4>
ErrorLogFormat "%{cu}t %M"

10
etc/vitrage/api-paste.ini
View File

@ -7,7 +7,7 @@ use = egg:Paste#urlmap
[composite:vitrage+keystone]
use = egg:Paste#urlmap
/ = vitrageversions_pipeline
/v1 = vitragev1_keystone_pipeline
/v1 = vitrage_event_pipline
/healthcheck = healthcheck
[composite:vitrage+keycloak]
@ -32,8 +32,8 @@ root = vitrage.api.controllers.root.VersionsController
[pipeline:vitragev1_noauth_pipeline]
pipeline = cors http_proxy_to_wsgi request_id osprofiler vitragev1
[pipeline:vitragev1_keystone_pipeline]
pipeline = cors http_proxy_to_wsgi request_id osprofiler keystoneauthtoken vitragev1
[pipeline:vitrage_event_pipline]
pipeline = cors http_proxy_to_wsgi request_id osprofiler basic_and_keystone_auth vitragev1
[pipeline:vitragev1_keycloak_pipeline]
pipeline = cors http_proxy_to_wsgi request_id osprofiler keycloakauthtoken vitragev1
@ -42,8 +42,8 @@ pipeline = cors http_proxy_to_wsgi request_id osprofiler keycloakauthtoken vitra
paste.app_factory = vitrage.api.app:app_factory
root = vitrage.api.controllers.v1.root.V1Controller
[filter:keystoneauthtoken]
paste.filter_factory = keystonemiddleware.auth_token:filter_factory
[filter:basic_and_keystone_auth]
paste.filter_factory = vitrage.middleware.basic_and_keystone_auth:filter_factory
oslo_config_project = vitrage
[filter:keycloakauthtoken]

13
vitrage/api/controllers/v1/event.py
View File

@ -18,8 +18,10 @@ from oslo_log import log
from osprofiler import profiler
from pecan.core import abort
from datetime import datetime
from vitrage.api.controllers.rest import RootRestController
from vitrage.api.policy import enforce
from vitrage.datasources.prometheus.driver import PROMETHEUS_EVENT_TYPE
LOG = log.getLogger(__name__)
@ -37,9 +39,14 @@ class EventController(RootRestController):
enforce("event post", pecan.request.headers,
pecan.request.enforcer, {})
event_time = kwargs['time']
event_type = kwargs['type']
details = kwargs['details']
prom_event_type = None
user_agent = pecan.request.headers.get('User-Agent')
if user_agent and user_agent.startswith("Alertmanager"):
prom_event_type = PROMETHEUS_EVENT_TYPE
event_time = kwargs.get('time', datetime.utcnow())
event_type = kwargs.get('type', prom_event_type)
details = kwargs.get('details', kwargs)
self.post_event(event_time, event_type, details)

3
vitrage/api_handler/apis/event.py
View File

@ -42,13 +42,12 @@ class EventApis(EntityGraphApisBase):
event = {EventProperties.TYPE: event_type,
EventProperties.TIME: event_time,
EventProperties.DETAILS: details}
if details.get(DoctorDetails.STATUS) == DoctorStatus.UP:
notification_type = DoctorProperties.CUSTOM_EVENT_UP
elif details.get(DoctorDetails.STATUS) == DoctorStatus.DOWN:
notification_type = DoctorProperties.CUSTOM_EVENT_DOWN
else:
raise Exception("Unknown status")
notification_type = event_type
self.oslo_notifier.info(
ctxt={'message_id': uuidutils.generate_uuid(),

101
vitrage/datasources/prometheus/driver.py
View File

@ -17,6 +17,7 @@ from oslo_log import log
from vitrage.common.constants import DatasourceAction
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EventProperties as EProps
from vitrage.datasources.alarm_driver_base import AlarmDriverBase
from vitrage.datasources.prometheus import PROMETHEUS_DATASOURCE
from vitrage.datasources.prometheus.properties import get_alarm_update_time
@ -68,23 +69,50 @@ class PrometheusDriver(AlarmDriverBase):
:param event: dictionary of this form:
{
"status": "firing",
"groupLabels": {
"alertname": "HighInodeUsage"
},
"groupKey": "{}:{alertname=\"HighInodeUsage\"}",
"commonAnnotations": {
"mount_point": "/%",
"description": "\"Consider ssh\"ing into the instance \"\n",
"title": "High number of inode usage",
"value": "96.81%",
"device": "/dev/vda1%",
"runbook": "troubleshooting/filesystem_alerts_inodes.md"
},
"alerts": [
"details":
{
"status": "firing",
"labels": {
"groupLabels": {
"alertname": "HighInodeUsage"
},
"groupKey": "{}:{alertname=\"HighInodeUsage\"}",
"commonAnnotations": {
"mount_point": "/%",
"description": "\"Consider ssh\"ing into instance \"\n",
"title": "High number of inode usage",
"value": "96.81%",
"device": "/dev/vda1%",
"runbook": "troubleshooting/filesystem_alerts_inodes.md"
},
"alerts": [
{
"status": "firing",
"labels": {
"severity": "critical",
"fstype": "ext4",
"instance": "localhost:9100",
"job": "node",
"alertname": "HighInodeUsage",
"device": "/dev/vda1",
"mountpoint": "/"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "http://devstack-4:9090/graph?g0.htm1",
"startsAt": "2018-05-03T12:25:38.231388525Z",
"annotations": {
"mount_point": "/%",
"description": "\"Consider ssh\"ing into instance\"\n",
"title": "High number of inode usage",
"value": "96.81%",
"device": "/dev/vda1%",
"runbook": "filesystem_alerts_inodes.md"
}
}
],
"version": "4",
"receiver": "vitrage",
"externalURL": "http://devstack-rocky-4:9093",
"commonLabels": {
"severity": "critical",
"fstype": "ext4",
"instance": "localhost:9100",
@ -92,32 +120,8 @@ class PrometheusDriver(AlarmDriverBase):
"alertname": "HighInodeUsage",
"device": "/dev/vda1",
"mountpoint": "/"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "http://devstack-rocky-4:9090/graph?g0.htm1",
"startsAt": "2018-05-03T12:25:38.231388525Z",
"annotations": {
"mount_point": "/%",
"description": "\"Consider ssh\"ing into the instance\"\n",
"title": "High number of inode usage",
"value": "96.81%",
"device": "/dev/vda1%",
"runbook": "troubleshooting/filesystem_alerts_inodes.md"
}
}
],
"version": "4",
"receiver": "vitrage",
"externalURL": "http://devstack-rocky-4:9093",
"commonLabels": {
"severity": "critical",
"fstype": "ext4",
"instance": "localhost:9100",
"job": "node",
"alertname": "HighInodeUsage",
"device": "/dev/vda1",
"mountpoint": "/"
}
}
:param event_type: The type of the event. Always 'prometheus.alarm'.
@ -128,18 +132,21 @@ class PrometheusDriver(AlarmDriverBase):
LOG.debug('Going to enrich event: %s', str(event))
alarms = []
details = event.get(EProps.DETAILS)
if details:
for alarm in details.get(PProps.ALERTS, []):
alarm[DSProps.EVENT_TYPE] = event_type
alarm[PProps.STATUS] = details[PProps.STATUS]
for alarm in event.get(PProps.ALERTS, []):
alarm[DSProps.EVENT_TYPE] = event_type
alarm[PProps.STATUS] = event[PProps.STATUS]
old_alarm = self._old_alarm(alarm)
alarm = self._filter_and_cache_alarm(alarm, old_alarm,
old_alarm = self._old_alarm(alarm)
alarm = \
self._filter_and_cache_alarm(alarm,
old_alarm,
self._filter_get_erroneous,
get_alarm_update_time(alarm))
if alarm:
alarms.append(alarm)
if alarm:
alarms.append(alarm)
LOG.debug('Enriched event. Created alarm events: %s', str(alarms))

138
vitrage/middleware/basic_and_keystone_auth.py Normal file
View File

@ -0,0 +1,138 @@
# Copyright 2018 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import werkzeug.http
from keystoneauth1.identity.generic import password
from keystoneauth1 import loading
from keystoneauth1 import session
from keystonemiddleware.auth_token import AuthProtocol
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import encodeutils
from six.moves import http_client as httplib
from webob import exc
LOG = logging.getLogger(__name__)
CFG_GROUP = "service_credentials"
class BasicAndKeystoneAuth(AuthProtocol):
def __init__(self, application, conf):
super(BasicAndKeystoneAuth, self).__init__(application, conf)
self.application = application
self.oslo_conf = cfg.ConfigOpts()
self.oslo_conf([],
project='vitrage',
validate_default_values=True)
password_option = loading.get_auth_plugin_conf_options('password')
self.oslo_conf.register_opts(password_option, group=CFG_GROUP)
self.auth_url = self.oslo_conf.service_credentials.auth_url
@property
def reject_auth_headers(self):
header_val = 'Keystone uri=\'%s\'' % self.auth_url
return [('WWW-Authenticate', header_val)]
def process_request(self, req):
"""Process request.
Evaluate the headers in a request and attempt to authenticate the
request according to authentication mode.
If the request comes through /v1/event api path then it can be
authenticate either with basic auth by providing username and
password or with keystone authentication.
If authenticated then additional headers are added to the
request for use by applications. If not authenticated the request
will be rejected or marked unauthenticated depending on
configuration.
"""
if req.path == '/v1/event':
basic_auth_info = self._get_basic_authenticator(req)
if basic_auth_info:
self._basic_authenticate(basic_auth_info, req)
else:
super(BasicAndKeystoneAuth, self).process_request(req)
else:
super(BasicAndKeystoneAuth, self).process_request(req)
def _basic_authenticate(self, auth_info, req):
try:
project_domain_id, project_name, user_domain_id = \
self._get_auth_params()
auth = password.Password(auth_url=self.auth_url,
username=auth_info.username,
password=auth_info.password,
user_domain_id=user_domain_id,
project_domain_id=project_domain_id,
project_name=project_name)
sess = session.Session(auth=auth)
token = sess.get_token()
project_id = str(auth.get_project_id(sess))
roles = str(auth.get_auth_ref(sess).role_names[0])
self._set_req_headers(req, token, project_id, roles)
except Exception as e:
to_unicode = encodeutils.exception_to_unicode(e)
message = 'Authorization exception: %s' % to_unicode
self._unauthorized(message)
def _get_auth_params(self):
user_domain_id = \
self.oslo_conf.service_credentials.user_domain_id
project_domain_id = \
self.oslo_conf.service_credentials.project_domain_id
project_name = self.oslo_conf.service_credentials.project_name
return project_domain_id, project_name, user_domain_id
def _unauthorized(self, message):
body = {'error': {
'code': httplib.UNAUTHORIZED,
'title': httplib.responses.get(httplib.UNAUTHORIZED),
'message': message,
}}
raise exc.HTTPUnauthorized(body=jsonutils.dumps(body),
headers=self.reject_auth_headers,
charset='UTF-8',
content_type='application/json')
@staticmethod
def _get_basic_authenticator(req):
auth = werkzeug.http.parse_authorization_header(
req.headers.get("Authorization"))
return auth
@staticmethod
def _set_req_headers(req, token, project_id, roles):
req.headers['X-Auth-Token'] = token
req.headers['X-Identity-Status'] = 'Confirmed'
req.headers['X-Project-Id'] = project_id
req.headers['X-Roles'] = roles
def filter_factory(global_conf, **local_conf):
"""Return a WSGI filter app for use with paste.deploy."""
conf = global_conf.copy()
conf.update(local_conf)
def auth_filter(app):
return BasicAndKeystoneAuth(app, conf)
return auth_filter

14
vitrage/service.py
View File

@ -27,6 +27,7 @@ LOG = log.getLogger(__name__)
def prepare_service(args=None, conf=None, config_files=None):
set_defaults()
if conf is None:
conf = cfg.ConfigOpts()
log.register_options(conf)
@ -59,3 +60,16 @@ def prepare_service(args=None, conf=None, config_files=None):
messaging.setup()
return conf
def set_defaults():
from oslo_middleware import cors
cfg.set_defaults(cors.CORS_OPTS,
allow_headers=[
'Authorization',
'X-Auth-Token',
'X-Subject-Token',
'X-User-Id',
'X-Domain-Id',
'X-Project-Id',
'X-Roles'])

189
vitrage/tests/functional/api/v1/test_basic.py Normal file
View File

@ -0,0 +1,189 @@
# Copyright 2018 - Nokia Corporation
# Copyright 2014 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# noinspection PyPackageRequirements
import uuid
from datetime import datetime
from mock import mock
from six.moves import http_client as httplib
from vitrage.tests.functional.api.v1 import FunctionalTest
HEADERS = {
'Authorization': 'Basic dml0cmFnZTpwYXNzd29yZA==',
'User-Agent': 'Alertmanager/0.15.0',
'Host': '127.0.0.1:8999',
'Content-Type': 'application/json'
}
EVENT_TYPE = 'prometheus.alarm'
VALID_TOKEN = uuid.uuid4().hex
PROJECT_ID = 'best_project'
class Role(object):
pass
ROLES = Role()
ROLES.role_names = ['admin']
EVENT_DETAILS = {
"status": "firing",
"groupLabels": {
"alertname": "HighInodeUsage"
},
"groupKey": "{}:{alertname=\"HighInodeUsage\"}",
"commonAnnotations": {
"mount_point": "/%",
"description": "\"Consider ssh\"ing into the instance \"\n",
"title": "High number of inode usage",
"value": "96.81%",
"device": "/dev/vda1%",
"runbook": "troubleshooting/filesystem_alerts_inodes.md"
},
"alerts": [
{
"status": "firing",
"labels": {
"severity": "critical",
"fstype": "ext4",
"instance": "localhost:9100",
"job": "node",
"alertname": "HighInodeUsage",
"device": "/dev/vda1",
"mountpoint": "/"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "http://devstack-rocky-4:9090/graph?g0.htm1",
"startsAt": "2018-05-03T12:25:38.231388525Z",
"annotations": {
"mount_point": "/%",
"description": "\"Consider ssh\"ing into the instance\"\n",
"title": "High number of inode usage",
"value": "96.81%",
"device": "/dev/vda1%",
"runbook": "troubleshooting/filesystem_alerts_inodes.md"
}
}
],
"version": "4",
"receiver": "vitrage",
"externalURL": "http://devstack-rocky-4:9093",
"commonLabels": {
"severity": "critical",
"fstype": "ext4",
"instance": "localhost:9100",
"job": "node",
"alertname": "HighInodeUsage",
"device": "/dev/vda1",
"mountpoint": "/"
}
}
ERR_MSG_MISSING_AUTH = 'The request you have made requires authentication.'
ERR_MSG_MISSING_VERSIONED_IDENTITY_ENDPOINTS = 'Authorization exception: ' \
'Could not find versioned ' \
'identity endpoints when ' \
'attempting to authenticate. ' \
'Please check that your ' \
'auth_url is correct.'
ERR_MISSING_AUTH = {'error': {
'code': httplib.UNAUTHORIZED,
'title': httplib.responses[httplib.UNAUTHORIZED],
'message': ERR_MSG_MISSING_AUTH,
}}
ERR_MISSING_VERSIONED_IDENTITY_ENDPOINTS = {'error': {
'code': httplib.UNAUTHORIZED,
'title': httplib.responses[httplib.UNAUTHORIZED],
'message': ERR_MSG_MISSING_VERSIONED_IDENTITY_ENDPOINTS,
}}
class BasicAuthTest(FunctionalTest):
def __init__(self, *args, **kwds):
super(BasicAuthTest, self).__init__(*args, **kwds)
self.auth = 'keystone'
keystoneauth__identity = 'keystoneauth1.identity'
@mock.patch('keystoneauth1.session.Session.get_token',
return_value=VALID_TOKEN)
@mock.patch('%s.base.BaseIdentityPlugin.get_project_id' %
keystoneauth__identity,
return_value=PROJECT_ID)
@mock.patch('%s.generic.base.BaseGenericPlugin.get_auth_ref' %
keystoneauth__identity,
return_value=ROLES)
@mock.patch('pecan.request')
def test_header_parsing(self, req_mock, *args):
resp = self.post_json('/event',
params={
'time': datetime.now().isoformat(),
'type': EVENT_TYPE,
'details': EVENT_DETAILS
},
headers=HEADERS)
req = resp.request
self.assertEqual('Confirmed', req.headers['X-Identity-Status'])
self.assertEqual(ROLES.role_names[0], req.headers['X-Roles'])
self.assertEqual(PROJECT_ID, req.headers['X-Project-Id'])
self.assertEqual(VALID_TOKEN, req.headers['X-Auth-Token'])
self.assertEqual(1, req_mock.client.call.call_count)
@mock.patch('keystoneauth1.session.Session.request')
def test_basic_mode_auth_wrong_authorization(self, *args):
wrong_headers = HEADERS.copy()
wrong_headers['Authorization'] = 'Basic dml0cmFnZTpwdml0cmFnZT=='
resp = self.post_json('/event',
params={
'time': datetime.now().isoformat(),
'type': EVENT_TYPE,
'details': EVENT_DETAILS
},
headers=wrong_headers,
expect_errors=True)
self.assertEqual(httplib.UNAUTHORIZED, resp.status_code)
self.assertDictEqual(ERR_MISSING_VERSIONED_IDENTITY_ENDPOINTS,
resp.json)
def test_in_basic_mode_auth_no_header(self):
resp = self.post_json('/event', expect_errors=True)
self.assertEqual(httplib.UNAUTHORIZED, resp.status_code)
self.assertDictEqual(ERR_MISSING_AUTH, resp.json)
@mock.patch('keystoneauth1.identity.generic.password.Password')
@mock.patch('keystoneauth1.session.Session')
@mock.patch('pecan.request')
def test_in_basic_mode_auth_success(self, req_mock, *args):
resp = self.post_json('/event',
params={
'time': datetime.now().isoformat(),
'type': EVENT_TYPE,
'details': EVENT_DETAILS
},
headers=HEADERS)
self.assertEqual(1, req_mock.client.call.call_count)
self.assertEqual(httplib.OK, resp.status_code)

81
vitrage/tests/resources/mock_configurations/driver/driver_prometheus_update_dynamic.json
View File

@ -1,21 +1,48 @@
{
"status": "firing",
"groupLabels": {
"alertname": "HighInodeUsage"
},
"groupKey": "{}:{alertname=\"HighInodeUsage\"}",
"commonAnnotations": {
"mount_point": "/%",
"description": "\"Consider ssh\"ing into the instance and removing files or clean\ntemp files\"\n",
"title": "High number of inode usage",
"value": "96.81%",
"device": "/dev/vda1%",
"runbook": "troubleshooting/filesystem_alerts_inodes.md"
},
"alerts": [
"details":
{
"status": "firing",
"labels": {
"groupLabels": {
"alertname": "HighInodeUsage"
},
"groupKey": "{}:{alertname=\"HighInodeUsage\"}",
"commonAnnotations": {
"mount_point": "/%",
"description": "\"Consider ssh\"ing into the instance and removing files or clean\ntemp files\"\n",
"title": "High number of inode usage",
"value": "96.81%",
"device": "/dev/vda1%",
"runbook": "troubleshooting/filesystem_alerts_inodes.md"
},
"alerts": [
{
"status": "firing",
"labels": {
"severity": "critical",
"fstype": "ext4",
"instance": "localhost:9100",
"job": "node",
"alertname": "HighInodeUsage",
"device": "/dev/vda1",
"mountpoint": "/"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "http://devstack-rocky-4:9090/graph?g0.expr=node_filesystem_files_free%7Bfstype%3D~%22%28ext.%7Cxfs%29%22%2Cjob%3D%22node%22%7D+%2F+node_filesystem_files%7Bfstype%3D~%22%28ext.%7Cxfs%29%22%2Cjob%3D%22node%22%7D+%2A+100+%3C%3D+100&g0.tab=1",
"startsAt": "2018-05-03T12:25:38.231388525Z",
"annotations": {
"mount_point": "/%",
"description": "\"Consider ssh\"ing into the instance and removing files or clean\ntemp files\"\n",
"title": "High number of inode usage",
"value": "96.81%",
"device": "/dev/vda1%",
"runbook": "troubleshooting/filesystem_alerts_inodes.md"
}
}
],
"version": "4",
"receiver": "vitrage",
"externalURL": "http://devstack-rocky-4:9093",
"commonLabels": {
"severity": "critical",
"fstype": "ext4",
"instance": "localhost:9100",
@ -23,30 +50,6 @@
"alertname": "HighInodeUsage",
"device": "/dev/vda1",
"mountpoint": "/"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "http://devstack-rocky-4:9090/graph?g0.expr=node_filesystem_files_free%7Bfstype%3D~%22%28ext.%7Cxfs%29%22%2Cjob%3D%22node%22%7D+%2F+node_filesystem_files%7Bfstype%3D~%22%28ext.%7Cxfs%29%22%2Cjob%3D%22node%22%7D+%2A+100+%3C%3D+100&g0.tab=1",
"startsAt": "2018-05-03T12:25:38.231388525Z",
"annotations": {
"mount_point": "/%",
"description": "\"Consider ssh\"ing into the instance and removing files or clean\ntemp files\"\n",
"title": "High number of inode usage",
"value": "96.81%",
"device": "/dev/vda1%",
"runbook": "troubleshooting/filesystem_alerts_inodes.md"
}
}
],
"version": "4",
"receiver": "vitrage",
"externalURL": "http://devstack-rocky-4:9093",
"commonLabels": {
"severity": "critical",
"fstype": "ext4",
"instance": "localhost:9100",
"job": "node",
"alertname": "HighInodeUsage",
"device": "/dev/vda1",
"mountpoint": "/"
}
}