diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bbb5722 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.pydevproject +.project +.settings +*.pyc +build +dist +*.egg-info +.tox +AUTHORS +ChangeLog diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..15dfa78 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2012, Datadog +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the Datadog nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL DATADOG BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..003e999 --- /dev/null +++ b/README.md @@ -0,0 +1,78 @@ +monasca-statsd +================ + +A Monasca-Statsd Python client. + +Quick Start Guide +----------------- + +First install the library with `pip` or `easy_install` + + # Install in system python ... + sudo pip install monasca-statsd + + # .. or into a virtual env + pip install monasca-statsd + +Then start instrumenting your code: + +``` python +# Import the module. +from monascastatsd import monascastatsd + +# Optionally, configure the host and port if you're running Statsd on a +# non-standard port. +monascastatsd.connect('localhost', 8125) + +# Increment a counter. +monascastatsd.increment('page.views') + +# Record a gauge 50% of the time. +monascastatsd.gauge('users.online', 123, sample_rate=0.5) + +# Sample a histogram. +monascastatsd.histogram('file.upload.size', 1234) + +# Time a function call. +@monascastatsd.timed('page.render') +def render_page(): + # Render things ... + +# Tag a metric. +monascastatsd.histogram('query.time', 10, dimensions = {"version": "1.0", "environment": "dev"}) +``` +Documentation +------------- + +Read the full API docs +[here](https://github.com/stackforge/monasca-statsd). + +Feedback +-------- + +To suggest a feature, report a bug, or general discussion, head over +[here](https://bugs.launchpad.net/monasca). + +Change Log +---------- +- 1.0.0 + - Initial version of the code + + +License +------- + +Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..a12991b --- /dev/null +++ b/Rakefile @@ -0,0 +1,11 @@ + +desc "Run tests" +task :test do + sh "nosetests" +end + +task :default => :test + +task :release => :test do + sh "python setup.py sdist upload" +end diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..8f5c25b --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,153 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/monasca-statsd.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/monasca-statsd.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/monasca-statsd" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/monasca-statsd" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..5b5c0a6 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,260 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -*- coding: utf-8 -*- +# +# monasca-statsd documentation build configuration file, created by +# sphinx-quickstart on Thu Jun 14 19:22:15 2012. +# +# This file is execfile()d with the current directory set to its containing dir +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ---------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'monasca-statsd' +copyright = u'2014, HP' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '1.0.0' +# The full version, including alpha/beta/rc tags. +release = '1.0.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all documents +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'monasca-statsd-pythondoc' + + +# -- Options for LaTeX output ------------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, +# documentclass [howto/manual]). +latex_documents = [ + ('index', + 'monasca-statsd-python.tex', + u'monasca-statsd-python Documentation', + u'HP', + 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output ------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'monasca-statsd-python', u'monasca-statsd-python Documentation', + [u'HP'], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ----------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'monasca-statsd-python', + u'monasca-statsd-python Documentation', + u'HP', + 'monasca-statsd-python', + 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..b358ffc --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,35 @@ +.. monasca-statsd documentation master file, created by + sphinx-quickstart on Thu Jun 14 19:22:15 2012. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +monasca-statsd +============== + + +.. automodule:: statsd + + .. autoclass:: Monasca-Statsd + :members: + +.. module:: statsd +.. data:: statsd + + A global :class:`~statsd.MonascaStatsd` instance that is easily shared + across an application's modules. Initialize this once in your application's + set-up code and then other modules can import and use it without further + configuration. + + >>> from monascastatsd import monascastatsd + >>> monascastatsd.connect(host='localhost', port=8125) + +Source +====== + +The Monasca-Statsd source is freely available on Github. Check it out `here +`_. + +Get in Touch +============ + +If you'd like to suggest a feature or report a bug, please add an issue `here `_. diff --git a/monascastatsd/__init__.py b/monascastatsd/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monascastatsd/monasca_statsd.py b/monascastatsd/monasca_statsd.py new file mode 100644 index 0000000..d095325 --- /dev/null +++ b/monascastatsd/monasca_statsd.py @@ -0,0 +1,207 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Monasca-Statsd is a Python client for Statsd that adds dimensions. +""" + +import functools +import logging +import random +import socket +import time + +try: + import itertools +except ImportError: + imap = map + + +log = logging.getLogger(__name__) + + +class MonascaStatsd(object): + + def __init__(self, host='localhost', port=8125, max_buffer_size=50): + """Initialize a MonascaStatsd object. + + >>> monascastatsd = MonascaStatsd() + + :param host: the host of the MonascaStatsd server. + :param port: the port of the MonascaStatsd server. + :param max_buffer_size: Maximum number of metric to buffer before + sending to the server if sending metrics in batch + """ + self._host = None + self._port = None + self.socket = None + self.max_buffer_size = max_buffer_size + self._send = self._send_to_server + self.connect(host, port) + self.encoding = 'utf-8' + + def __enter__(self): + self.open_buffer(self.max_buffer_size) + return self + + def __exit__(self, the_type, value, traceback): + self.close_buffer() + + def open_buffer(self, max_buffer_size=50): + '''Open a buffer to send a batch of metrics in one packet + + You can also use this as a context manager. + + >>> with DogStatsd() as batch: + >>> batch.gauge('users.online', 123) + >>> batch.gauge('active.connections', 1001) + + ''' + self.max_buffer_size = max_buffer_size + self.buffer = [] + self._send = self._send_to_buffer + + def close_buffer(self): + '''Flush the buffer and switch back to single metric packets.''' + self._send = self._send_to_server + self._flush_buffer() + + def connect(self, host, port): + """Connect to the monascastatsd server on the given host and port.""" + self._host = host + self._port = int(port) + self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.socket.connect((self._host, self._port)) + + def gauge(self, metric, value, dimensions=None, sample_rate=1): + """Record the value of a gauge, optionally setting a list of + + dimensions and a sample rate. + + >>> monascastatsd.gauge('users.online', 123) + >>> monascastatsd.gauge('active.connections', 1001, + >>> dimensions={"protocol": "http"}) + """ + return self._report(metric, 'g', value, dimensions, sample_rate) + + def increment(self, metric, value=1, dimensions=None, sample_rate=1): + """Increment a counter, optionally setting a value, dimensions + + and a sample rate. + + >>> monascastatsd.increment('page.views') + >>> monascastatsd.increment('files.transferred', 124) + """ + self._report(metric, 'c', value, dimensions, sample_rate) + + def decrement(self, metric, value=1, dimensions=None, sample_rate=1): + """Decrement a counter, optionally setting a value, dimensions and a + + sample rate. + + >>> monascastatsd.decrement('files.remaining') + >>> monascastatsd.decrement('active.connections', 2) + """ + self._report(metric, 'c', -value, dimensions, sample_rate) + + def histogram(self, metric, value, dimensions=None, sample_rate=1): + """Sample a histogram value, optionally setting dimensions and a + + sample rate. + + >>> monascastatsd.histogram('uploaded.file.size', 1445) + >>> monascastatsd.histogram('album.photo.count', 26, + >>> dimensions={"gender": "female"}) + """ + self._report(metric, 'h', value, dimensions, sample_rate) + + def timing(self, metric, value, dimensions=None, sample_rate=1): + """Record a timing, optionally setting dimensions and a sample rate. + + >>> monascastatsd.timing("query.response.time", 1234) + """ + self._report(metric, 'ms', value, dimensions, sample_rate) + + def timed(self, metric, dimensions=None, sample_rate=1): + """A decorator that will measure the distribution of a function's + + run time. Optionally specify a list of tag or a sample rate. + :: + + @monascastatsd.timed('user.query.time', sample_rate=0.5) + def get_user(user_id): + # Do what you need to ... + pass + + # Is equivalent to ... + start = time.time() + try: + get_user(user_id) + finally: + monascastatsd.timing('user.query.time', time.time() - start) + """ + def wrapper(func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + start = time.time() + result = func(*args, **kwargs) + self.timing(metric, + time.time() - start, + dimensions=dimensions, + sample_rate=sample_rate) + return result + wrapped.__name__ = func.__name__ + wrapped.__doc__ = func.__doc__ + wrapped.__dict__.update(func.__dict__) + return wrapped + return wrapper + + def set(self, metric, value, dimensions=None, sample_rate=1): + """Sample a set value. + + >>> monascastatsd.set('visitors.uniques', 999) + """ + + self._report(metric, 's', value, dimensions, sample_rate) + + def _report(self, metric, metric_type, value, dimensions, sample_rate): + if sample_rate != 1 and random.random() > sample_rate: + return + + payload = [metric, ":", value, "|", metric_type] + if sample_rate != 1: + payload.extend(["|@", sample_rate]) + if dimensions: + payload.extend(["|#"]) + payload.append(dimensions) + + encoded = "".join(itertools.imap(str, payload)) + self._send(encoded) + + def _send_to_server(self, packet): + try: + self.socket.send(packet.encode(self.encoding)) + except socket.error: + log.exception("Error submitting metric") + + def _send_to_buffer(self, packet): + self.buffer.append(packet) + if len(self.buffer) >= self.max_buffer_size: + self._flush_buffer() + + def _flush_buffer(self): + self._send_to_server("\n".join(self.buffer)) + self.buffer = [] + +monascastatsd = MonascaStatsd() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7cbcac0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +ConfigParser +python-statsd diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..a870ace --- /dev/null +++ b/setup.cfg @@ -0,0 +1,37 @@ +[metadata] +name=monasca-statsd +maintainer=monasca +maintainer_email=monasca@lists.launchpad.net +description-file = README.md +home-page = https://github.com/stackforge/monasca-statsd +keywords= + openstack + monitoring + statsd +license=Apache-2 +include_package_data=True +test_suite=nose.collector +classifier= + Development Status :: 5 - Production/Stable + License :: OSI Approved :: Apache Software License + Topic :: System :: Monitoring + +[files] +packages = + monascastatsd + statsd-generator + +[global] +setup-hooks = + pbr.hooks.setup_hook + +[pbr] +autodoc_index_modules = True + +[wheel] +universal = 1 + +[egg_info] +tag_build = +tag_date = 0 +tag_svn_revision = 0 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..a92d122 --- /dev/null +++ b/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['pbr'], + pbr=True +) diff --git a/statsd-generator/README.md b/statsd-generator/README.md new file mode 100644 index 0000000..ec1279c --- /dev/null +++ b/statsd-generator/README.md @@ -0,0 +1,20 @@ +# Python Monasca-statsd Generator + +This test tool will create monasca-statsd metrics and send them to the monitoring agent. +To use it, you will need to install the following python libraries: +``` +sudo apt-get install python-setuptools +sudo pip install monasca-statsd +``` + +## Usage +### To run the generator: +``` +1) edit the config file (generator.conf) and set the target host, port, number of +iterations and delay (in seconds) between iterations. +2) cd to the statsd-generator directory +3) Type ./generator.py +4) The tool will send 4 different types of monasca-statsd messages to the monitoring +agent and then sleep for the duration of the delay and then will continue to the +next iteration until the number of iterations is reached. +``` \ No newline at end of file diff --git a/statsd-generator/__init__.py b/statsd-generator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/statsd-generator/generator.conf b/statsd-generator/generator.conf new file mode 100644 index 0000000..6105fc9 --- /dev/null +++ b/statsd-generator/generator.conf @@ -0,0 +1,5 @@ +[main] +iterations: 5 +delay: 3 +host: localhost +port: 8125 diff --git a/statsd-generator/generator.py b/statsd-generator/generator.py new file mode 100755 index 0000000..e7d5d6a --- /dev/null +++ b/statsd-generator/generator.py @@ -0,0 +1,100 @@ +#!/usr/bin/python +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ConfigParser +import random +import time + +import monascastatsd.monasca_statsd + +import statsd + +'''The statsd generator will provide a testbed for testing + +the monasca agent statsd capability with dimensions +''' + + +class MonascaStatsdGenerator(object): + + '''Class to generate MonascaStatsd metrics.''' + + def __init__(self, config): + '''Constructor.''' + print (config) + self.num_of_iterations = int(config["iterations"]) + self.delay = int(config["delay"]) + self.host = config["host"] + self.port = config["port"] + + def send_messages(self): + '''Main processing for sending messages.''' + try: + mstatsd = monascastatsd.monasca_statsd.MonascaStatsd() + mstatsd.connect(self.host, self.port) + for index in range(1, self.num_of_iterations + 1): + print("Starting iteration " + str(index) + + " of " + str(self.num_of_iterations)) + mstatsd.increment('Teraflops', 5) + mstatsd.gauge('NumOfTeraflops', + random.uniform(1.0, 10.0), + dimensions={'Origin': 'Dev', + 'Environment': 'Test'}) + mstatsd.histogram('file.upload.size', + random.randrange(1, 100), + dimensions={'Version': '1.0'}) + # Send some regular statsd messages + connection = statsd.Connection() + counter = statsd.Counter('statsd_generator') + counter += 1 + gauge = statsd.Gauge('statsd_generator') + gauge.send('cpu_percent', + random.uniform(1.0, 100.0)) + average = statsd.Average('statsd_generator', connection) + average.send('cpu_avg', random.randrange(1, 100)) + print("Completed iteration " + str(index) + + ". Sleeping for " + str(self.delay) + " seconds...") + time.sleep(self.delay) + except Exception: + print ("Error sending statsd messages...") + raise + + +def read_config(): + '''Read in the config file.''' + config = ConfigParser.ConfigParser() + config.read("generator.conf") + config_options = {} + section = "main" + options = config.options(section) + for option in options: + try: + config_options[option] = config.get(section, option) + if config_options[option] == -1: + print("skip: %s" % option) + except Exception: + print("exception on %s!" % option) + config_options[option] = None + return config_options + + +def main(): + config = read_config() + generator = MonascaStatsdGenerator(config) + generator.send_messages() + +if __name__ == "__main__": + main() diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..c0de34d --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1 @@ +hacking>=0.9.2,<0.10 diff --git a/tests/test_monascastatsd.py b/tests/test_monascastatsd.py new file mode 100644 index 0000000..995b3b2 --- /dev/null +++ b/tests/test_monascastatsd.py @@ -0,0 +1,202 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -*- coding: utf-8 -*- + +"""Tests for monascastatsd.py.""" + +import collections +import socket +import time +import unittest + +import monascastatsd.monasca_statsd + + +class FakeSocket(object): + + """A fake socket for testing.""" + + def __init__(self): + self.payloads = collections.deque() + + def send(self, payload): + self.payloads.append(payload) + + def recv(self): + try: + return self.payloads.popleft() + except IndexError: + return None + + def __repr__(self): + return str(self.payloads) + + +class BrokenSocket(FakeSocket): + + def send(self, payload): + raise socket.error("Socket error") + + +class TestMonStatsd(unittest.TestCase): + + def setUp(self): + self.monascastatsd = monascastatsd.monasca_statsd.MonascaStatsd() + self.monascastatsd.socket = FakeSocket() + + def recv(self): + return self.monascastatsd.socket.recv() + + def test_set(self): + self.monascastatsd.set('set', 123) + assert self.recv() == 'set:123|s' + + def test_gauge(self): + self.monascastatsd.gauge('gauge', 123.4) + assert self.recv() == 'gauge:123.4|g' + + def test_counter(self): + self.monascastatsd.increment('page.views') + self.assertEqual('page.views:1|c', self.recv()) + + self.monascastatsd.increment('page.views', 11) + self.assertEqual('page.views:11|c', self.recv()) + + self.monascastatsd.decrement('page.views') + self.assertEqual('page.views:-1|c', self.recv()) + + self.monascastatsd.decrement('page.views', 12) + self.assertEqual('page.views:-12|c', self.recv()) + + def test_histogram(self): + self.monascastatsd.histogram('histo', 123.4) + self.assertEqual('histo:123.4|h', self.recv()) + + def test_gauge_with_dimensions(self): + self.monascastatsd.gauge('gt', 123.4, + dimensions=['country:china', + 'age:45', + 'color:blue']) + self.assertEqual("gt:123.4|g|#[" + + "'country:china', " + + "'age:45', " + + "'color:blue']", + self.recv()) + + def test_counter_with_dimensions(self): + self.monascastatsd.increment('ct', + dimensions=['country:canada', + 'color:red']) + self.assertEqual("ct:1|c|#['country:canada', 'color:red']", + self.recv()) + + def test_histogram_with_dimensions(self): + self.monascastatsd.histogram('h', 1, dimensions=['color:red']) + self.assertEqual("h:1|h|#['color:red']", self.recv()) + + def test_sample_rate(self): + self.monascastatsd.increment('c', sample_rate=0) + assert not self.recv() + for _ in range(10000): + self.monascastatsd.increment('sampled_counter', sample_rate=0.3) + self.assert_almost_equal(3000, + len(self.monascastatsd.socket.payloads), + 150) + self.assertEqual('sampled_counter:1|c|@0.3', self.recv()) + + def test_samples_with_dimensions(self): + for _ in range(100): + self.monascastatsd.gauge('gst', + 23, + dimensions=['status:sampled'], + sample_rate=0.9) + + def test_samples_with_dimensions(self): + for _ in range(100): + self.monascastatsd.gauge('gst', + 23, + dimensions=['status:sampled'], + sample_rate=0.9) + self.assertEqual('gst:23|g|@0.9|#status:sampled') + + def test_timing(self): + self.monascastatsd.timing('t', 123) + self.assertEqual('t:123|ms', self.recv()) + + @staticmethod + def assert_almost_equal(a, b, delta): + assert 0 <= abs(a - b) <= delta, "%s - %s not within %s" % (a, + b, + delta) + + def test_socket_error(self): + self.monascastatsd.socket = BrokenSocket() + self.monascastatsd.gauge('no error', 1) + assert True, 'success' + + def test_timed(self): + + @self.monascastatsd.timed('timed.test') + def func(a, b, c=1, d=1): + """docstring.""" + time.sleep(0.5) + return (a, b, c, d) + + self.assertEqual('func', func.__name__) + self.assertEqual('docstring.', func.__doc__) + + result = func(1, 2, d=3) + # Assert it handles args and kwargs correctly. + self.assertEqual(result, (1, 2, 1, 3)) + + packet = self.recv() + name_value, type_ = packet.split('|') + name, value = name_value.split(':') + + self.assertEqual('ms', type_) + self.assertEqual('timed.test', name) + self.assert_almost_equal(0.5, float(value), 0.1) + + def test_batched(self): + self.monascastatsd.open_buffer() + self.monascastatsd.gauge('page.views', 123) + self.monascastatsd.timing('timer', 123) + self.monascastatsd.close_buffer() + + self.assertEqual('page.views:123|g\ntimer:123|ms', self.recv()) + + def test_context_manager(self): + fake_socket = FakeSocket() + with monascastatsd.monasca_statsd.MonascaStatsd() as statsd: + statsd.socket = fake_socket + statsd.gauge('page.views', 123) + statsd.timing('timer', 123) + + self.assertEqual('page.views:123|g\ntimer:123|ms', fake_socket.recv()) + + def test_batched_buffer_autoflush(self): + fake_socket = FakeSocket() + with monascastatsd.monasca_statsd.MonascaStatsd() as statsd: + statsd.socket = fake_socket + for _ in range(51): + statsd.increment('mycounter') + self.assertEqual('\n'.join(['mycounter:1|c' for _ in range(50)]), + fake_socket.recv()) + + self.assertEqual('mycounter:1|c', fake_socket.recv()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..92a84af --- /dev/null +++ b/tox.ini @@ -0,0 +1,24 @@ +[tox] +envlist = py27,pep8 +minversion = 1.6 +skipsdist = True + +[testenv] +usedevelop = True +install_command = pip install -U {opts} {packages} +setenv = + VIRTUAL_ENV={envdir} + DISCOVER_DIRECTORY=tests +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +whitelist_externals = bash + +[testenv:pep8] +commands = flake8 + +[testenv:venv] +commands = {posargs} + +[flake8] +show-source = True +exclude=.venv,.git,.tox,dist,*egg,build