openstack
/
monasca-notification
Code Issues Proposed changes

Switch to using monasca-statsd 

Switch to monasca-statsd so we can send additional dimensions
for service and component with the statsd messages.

Change-Id: Ic6ff3b67b4148c070ec9eec9f9f990680b5e9f4c
This commit is contained in:
gary-hessler 2014-10-22 15:04:32 -06:00
parent 2fea98ec2b
commit 9025d8b0e4
8 changed files with 56 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -3,3 +3,7 @@
.tox
AUTHORS
ChangeLog
.*project
build
dist
monasca_notification.egg-info

18
monasca_notification/processors/alarm_processor.py
View File

@ -15,9 +15,8 @@
import json
import logging
import multiprocessing
import monascastatsd as mstatsd
import MySQLdb
import statsd
import time
from monasca_notification.notification import Notification
@ -36,7 +35,8 @@ class AlarmProcessor(BaseProcessor):
self.alarm_ttl = alarm_ttl
self.notification_queue = notification_queue
self.finished_queue = finished_queue
self.monascastatsd = mstatsd.Client(name='monasca',
dimensions=BaseProcessor.dimensions)
try:
self.mysql = MySQLdb.connect(host=mysql_host, user=mysql_user, passwd=mysql_passwd, db=dbname)
self.mysql.autocommit(True)
@ -88,12 +88,12 @@ class AlarmProcessor(BaseProcessor):
"""Check the notification setting for this project in mysql then create the appropriate notification or
add to the finished_queue
"""
failed_parse_count = self.monascastatsd.get_counter(name='alarms_failed_parse_count')
no_notification_count = self.monascastatsd.get_counter(name='alarms_no_notification_count')
notification_count = self.monascastatsd.get_counter(name='created_count')
db_time = self.monascastatsd.get_timer()
cur = self.mysql.cursor()
pname = multiprocessing.current_process().name
failed_parse_count = statsd.Counter('AlarmsFailedParse-%s' % pname)
no_notification_count = statsd.Counter('AlarmsNoNotification-%s' % pname)
notification_count = statsd.Counter('NotificationsCreated-%s' % pname)
db_time = statsd.Timer('ConfigDBTime-%s' % pname)
while True:
raw_alarm = self.alarm_queue.get()
@ -116,7 +116,7 @@ class AlarmProcessor(BaseProcessor):
continue
try:
with db_time.time():
with db_time.time('config_db_time'):
cur.execute("""SELECT name, type, address
FROM alarm_action as aa
JOIN notification_method as nm ON aa.action_id = nm.id

3
monasca_notification/processors/base.py
View File

@ -19,6 +19,9 @@ log = logging.getLogger(__name__)
class BaseProcessor(object):
dimensions = {'service': 'monitoring', 'component': 'monasca-notification'}
@staticmethod
def _add_to_queue(queue, queue_name, msg):
"""Warns on full queue then does a blocking push to the queue.

10
monasca_notification/processors/kafka_consumer.py
View File

@ -17,7 +17,7 @@ import kafka.client
import kafka.common
import kafka.consumer
import logging
import statsd
import monascastatsd as mstatsd
from monasca_notification.processors.base import BaseProcessor
@ -26,6 +26,7 @@ log = logging.getLogger(__name__)
class KafkaConsumer(BaseProcessor):
"""Pull from the alarm topic and place alarm objects on the sent_queue.
No commit is being done until processing is finished and as the processing can take some time it is done in
another step.
@ -43,6 +44,8 @@ class KafkaConsumer(BaseProcessor):
# No auto-commit so that commits only happen after the alarm is processed.
self.consumer = kafka.consumer.SimpleConsumer(self.kafka, group, topic, auto_commit=False)
self.consumer.provide_partition_info() # Without this the partition is not provided in the response
self.monascastatsd = mstatsd.Client(name='monasca',
dimensions=BaseProcessor.dimensions)
self._initialize_offsets(group, topic)
# After my pull request is merged I can remove _initialize_offsets and use
@ -84,10 +87,11 @@ class KafkaConsumer(BaseProcessor):
def run(self):
"""Consume from kafka and place alarm objects on the sent_queue
"""
counter = statsd.Counter('ConsumedFromKafka')
consumed_from_kafka = self.monascastatsd.get_counter(name='consumed_from_kafka')
try:
for message in self.consumer:
counter += 1
consumed_from_kafka += 1
log.debug("Consuming message from kafka, partition %d, offset %d" % (message[0], message[1].offset))
self._add_to_queue(self.queue, 'alarms', message)
except Exception:

25
monasca_notification/processors/notification_processor.py
View File

@ -15,9 +15,8 @@
import email.mime.text
import logging
import multiprocessing
import monascastatsd as mstatsd
import smtplib
import statsd
import time
from monasca_notification.processors.base import BaseProcessor
@ -38,6 +37,8 @@ class NotificationProcessor(BaseProcessor):
# Types as key, method used to process that type as value
self.notification_types = {'email': self._send_email}
self.monascastatsd = mstatsd.Client(name='monasca',
dimensions=BaseProcessor.dimensions)
def _send_email(self, notification):
"""Send the notification via email
@ -86,15 +87,10 @@ class NotificationProcessor(BaseProcessor):
For each notification in a message it is sent according to its type.
If all notifications fail the alarm partition/offset are added to the the finished queue
"""
pname = multiprocessing.current_process().name
invalid_count = statsd.Counter('NotificationsInvalidType-%s' % pname)
failed_count = statsd.Counter('NotificationsSentFailed-%s' % pname)
smtp_sent_count = statsd.Counter('NotificationsSentSMTP-%s' % pname)
counters = {'email': smtp_sent_count}
smtp_time = statsd.Timer('SMTPTime-%s' % pname)
timers = {'email': smtp_time}
counters = {'email': self.monascastatsd.get_counter(name='sent_smtp_count')}
timers = {'email': self.monascastatsd.get_timer()}
invalid_type_count = self.monascastatsd.get_counter(name='invalid_type_count')
sent_failed_count = self.monascastatsd.get_counter(name='sent_failed_count')
while True:
notifications = self.notification_queue.get()
@ -102,13 +98,14 @@ class NotificationProcessor(BaseProcessor):
for notification in notifications:
if notification.type not in self.notification_types:
log.warn('Notification type %s is not a valid type' % notification.type)
invalid_count += 1
invalid_type_count += 1
continue
else:
with timers[notification.type].time():
with timers[notification.type].time('smtp_time'):
sent = self.notification_types[notification.type](notification)
if sent is None:
failed_count += 1
sent_failed_count += 1
else:
sent.notification_timestamp = time.time()
sent_notifications.append(sent)

10
monasca_notification/processors/sent_notification_processor.py
View File

@ -16,7 +16,7 @@
import kafka.client
import kafka.producer
import logging
import statsd
import monascastatsd as mstatsd
from monasca_notification.processors.base import BaseProcessor
@ -39,6 +39,8 @@ class SentNotificationProcessor(BaseProcessor):
self.topic = topic
self.finished_queue = finished_queue
self.sent_queue = sent_queue
self.monascastatsd = mstatsd.Client(name='monasca',
dimensions=BaseProcessor.dimensions)
self.kafka = kafka.client.KafkaClient(url)
self.producer = kafka.producer.SimpleProducer(
@ -52,12 +54,14 @@ class SentNotificationProcessor(BaseProcessor):
"""Takes messages from the sent_queue, puts them on the kafka notification topic and then adds
partition/offset to the finished queue
"""
published_count = statsd.Counter('PublishedToKafka')
published_to_kafka = self.monascastatsd.get_counter(name='published_to_kafka')
while True:
notifications = self.sent_queue.get()
for notification in notifications:
responses = self.producer.send_messages(self.topic, notification.to_json())
published_count += 1
published_to_kafka += 1
log.debug('Published to topic %s, message %s' % (self.topic, notification.to_json()))
for resp in responses:
if resp.error != 0:

22
monasca_notification/state_tracker.py
View File

@ -20,10 +20,10 @@ import kazoo.client
import kazoo.exceptions
import logging
import Queue
import statsd
import time
from monasca_notification import notification_exceptions
import monascastatsd as mstatsd
log = logging.getLogger(__name__)
@ -61,13 +61,16 @@ class KafkaStateTracker(object):
self.lock_path = '/locks/monasca-notification/%s' % topic
self._offsets = None # Initialized in the beginning of the run
self._dimensions = {'service': 'monitoring', 'component': 'monasca-notification'}
# This is a dictionary of sets used for tracking finished offsets when there is a gap and the committed offset
# can not yet be advanced
self._uncommitted_offsets = collections.defaultdict(set)
self._last_commit_time = collections.defaultdict(time.time)
self.zk_timer = statsd.Timer('OffsetCommitTime')
self.offset_update_count = statsd.Counter('AlarmsOffsetUpdated')
monascastatsd = mstatsd.Client(name='monasca',
dimensions=self._dimensions)
self.offset_update_count = monascastatsd.get_counter(name='alarms_offset_update_count')
self.finished_count = monascastatsd.get_counter(name='alarms_finished_count')
self.kafka_timer = monascastatsd.get_timer()
def _drop_lock(self):
"""Drop the lock file kept in zookeeper
@ -140,7 +143,6 @@ class KafkaStateTracker(object):
if not self.has_lock:
raise notification_exceptions.NotificationException('Attempt to begin run without Zookeeper Lock')
finished_count = statsd.Counter('AlarmsFinished')
while True:
# If self.stop is True run the queue until it is empty, do final commits then exit
if self.stop and self.finished_queue.empty():
@ -161,7 +163,8 @@ class KafkaStateTracker(object):
except Queue.Empty:
continue # This is non-blocking so the self.stop signal has a chance to take affect
finished_count += 1
self.finished_count += 1
partition = int(msg[0])
offset = int(msg[1])
@ -203,12 +206,15 @@ class KafkaStateTracker(object):
self.offsets
self.offset_update_count += value - self._offsets[partition]
self._offsets[partition] = value
req = kafka.common.OffsetCommitRequest(self.topic, partition, value, None)
try:
responses = self.kafka.send_offset_commit_request(self.kafka_group, [req])
kafka.common.check_error(responses[0])
with self.kafka_timer.time('offset_commit_time_sec'):
responses = self.kafka.send_offset_commit_request(self.kafka_group, [req])
kafka.common.check_error(responses[0])
log.debug('Updated committed offset for partition %s, offset %s' % (partition, value))
except kafka.common.KafkaError:
log.exception('Error updating the committed offset in kafka, partition %s, value %s' % (partition, value))

2
requirements.txt
View File

@ -2,5 +2,5 @@ kafka-python>=0.9.1
kazoo>=1.3
MySQL-python
pbr>=0.6,<1.0
python-statsd>=1.6.3
monasca-statsd
PyYAML