From c43115400e2edabcc765c666c5dbd4ba756db0c8 Mon Sep 17 00:00:00 2001 From: Dina Belova Date: Fri, 1 Aug 2014 17:19:00 +0400 Subject: [PATCH] Initial OpenTSDB REST (v2.0) client implimentation Change-Id: I36faca3c18d3b19aa9c32b8a4d396e4351dc8726 --- .gitignore | 10 ++ .testr.conf | 4 + LICENSE | 175 ++++++++++++++++++++++++++++ MANIFEST.in | 5 + README.rst | 6 + opentsdbclient/__init__.py | 32 +++++ opentsdbclient/client.py | 121 +++++++++++++++++++ opentsdbclient/tests/__init__.py | 24 ++++ opentsdbclient/tests/test_client.py | 66 +++++++++++ opentsdbclient/utils.py | 23 ++++ requirements.txt | 2 + setup.cfg | 28 +++++ setup.py | 29 +++++ test-requirements.txt | 5 + tox.ini | 36 ++++++ 15 files changed, 566 insertions(+) create mode 100644 .gitignore create mode 100644 .testr.conf create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 README.rst create mode 100644 opentsdbclient/__init__.py create mode 100644 opentsdbclient/client.py create mode 100644 opentsdbclient/tests/__init__.py create mode 100644 opentsdbclient/tests/test_client.py create mode 100644 opentsdbclient/utils.py create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 test-requirements.txt create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6e1392b --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*.pyc +*.egg-info +build +.tox +.venv +.testr +.testrepository +AUTHORS +ChangeLog +*.egg \ No newline at end of file diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 0000000..9408715 --- /dev/null +++ b/.testr.conf @@ -0,0 +1,4 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ./ $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..67db858 --- /dev/null +++ b/LICENSE @@ -0,0 +1,175 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..d9efa58 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +include AUTHORS +include LICENSE +include README.md +include ChangeLog +include tox.ini diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..6dd00b0 --- /dev/null +++ b/README.rst @@ -0,0 +1,6 @@ +Welcome to simple OpenTSDB REST API client +========================================== + +This client uses OpenTSDB v2.0 REST API to communicate with database. It aims +to provide simple way of posting metrics to the time-series DB and asking +for listing them using built-in query language. \ No newline at end of file diff --git a/opentsdbclient/__init__.py b/opentsdbclient/__init__.py new file mode 100644 index 0000000..11b7754 --- /dev/null +++ b/opentsdbclient/__init__.py @@ -0,0 +1,32 @@ +# Copyright 2014: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class OpenTSDBError(Exception): + def __init__(self, msg=None): + if msg is None: + msg = 'Unknown OpenTSDB error occurred. \n %s \n %s' + super(OpenTSDBError, self).__init__(msg) + + +class InvalidOpenTSDBFormat(OpenTSDBError): + """Error raised when data posted to OpenTSDB has the invalid format.""" + def __init__(self, actual, expected=None): + msg = 'Data %s has the unexpected by openTSDB format.' % actual + if expected: + msg += ' Please provide data in %s format.' + super(InvalidOpenTSDBFormat, self).__init__(msg) + self.actual = actual + self.expected = expected \ No newline at end of file diff --git a/opentsdbclient/client.py b/opentsdbclient/client.py new file mode 100644 index 0000000..5eaa855 --- /dev/null +++ b/opentsdbclient/client.py @@ -0,0 +1,121 @@ +# Copyright 2014: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +import requests + +import opentsdbclient +from opentsdbclient import utils + + +class OpenTSDBClient(object): + def __init__(self, opentsdb_host, opentsdb_port): + self.host = opentsdb_host + self.port = opentsdb_port + + def get_statistics(self): + """Get info about what metrics are registered and with what stats.""" + req = requests.get(utils.STATS_TEMPL % {'host': self.host, + 'port': self.port}) + return req + + def put_meter(self, meters): + """Post new meter(s) to the database. + + Meter dictionary *should* contain the following four required fields: + - metric: the name of the metric you are storing + - timestamp: a Unix epoch style timestamp in seconds or milliseconds. + The timestamp must not contain non-numeric characters. + - value: the value to record for this data point. It may be quoted or + not quoted and must conform to the OpenTSDB value rules. + - tags: a map of tag name/tag value pairs. At least one pair must be + supplied. + """ + res = [] + if type(meters) == dict: + meters = [meters] + for meter_dict in meters: + if (set(meter_dict.keys()) + != set(['metric', 'timestamp', 'value', 'tags'])): + raise opentsdbclient.InvalidOpenTSDBFormat( + actual=meter_dict, + expected="{'metric': , 'timestamp': , " + "'value': , 'tags': }") + + req = requests.post(utils.PUT_TEMPL % + {'host': self.host, 'port': self.port}, + data=json.dumps(meter_dict)) + res.append(req) + return res + + def define_retention(self, tsuid, retention_days): + """Set retention days for the defined by ID timeseries. + + ########################################################## + NOTE: currently not working directly through the REST API. + that should be done directly on the HBase level. + ########################################################## + + :param tsuid: hexadecimal representation of the timeseries UID + :param retention_days: number of days of data points to retain for the + given timeseries. When set to 0, the default, + data is retained indefinitely. + """ + meta_data = {'tsuid': tsuid, 'retention': retention_days} + req = requests.post(utils.META_TEMPL % {'host': self.host, + 'port': self.port, + 'tsuid': tsuid}, + data=json.dumps(meta_data)) + return req + + def get_aggregators(self): + """Used to get the list of default aggregation functions.""" + req = requests.get(utils.AGGR_TEMPL % {'host': self.host, + 'port': self.port}) + return req + + def get_version(self): + """Used to check OpenTSDB version. + + That might be needed in case of unknown bugs - this code is written + only for the 2.x REST API version, so some of the failures might refer + to the wrong OpenTSDB version installed. + """ + req = requests.get(utils.VERSION_TEMPL % {'host': self.host, + 'port': self.port}) + return req + + def _make_query(self, query, verb): + meth = getattr(requests, verb.lower(), None) + if meth is None: + pass + req = meth(utils.QUERY_TEMPL % {'host': self.host, 'port': self.port, + 'query': query}) + return req + + def get_query(self, query): + return self._make_query(query, 'get') + + def process_response(self, resp): + try: + res = json.loads(resp.text) + except Exception: + raise opentsdbclient.OpenTSDBError(resp.text) + + if 'errors' in res: + raise opentsdbclient.OpenTSDBError(res['error']) + + return res diff --git a/opentsdbclient/tests/__init__.py b/opentsdbclient/tests/__init__.py new file mode 100644 index 0000000..006eb95 --- /dev/null +++ b/opentsdbclient/tests/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2014: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testtools + + +class BaseTestCase(testtools.TestCase): + + def setUp(self): + super(BaseTestCase, self).setUp() + self.host = '127.0.0.1' + self.port = 4242 diff --git a/opentsdbclient/tests/test_client.py b/opentsdbclient/tests/test_client.py new file mode 100644 index 0000000..fe96882 --- /dev/null +++ b/opentsdbclient/tests/test_client.py @@ -0,0 +1,66 @@ +# Copyright 2014: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +import mock +import requests + +from opentsdbclient import client +from opentsdbclient import tests + + +class ClientTest(tests.BaseTestCase): + def setUp(self): + super(ClientTest, self).setUp() + self.client = client.OpenTSDBClient(opentsdb_host=self.host, + opentsdb_port=self.port) + + @mock.patch.object(requests, 'get') + def test_get_statistics(self, get_mock): + self.client.get_statistics() + get_mock.assert_called_once_with('http://127.0.0.1:4242/api/stats') + + @mock.patch.object(requests, 'post') + def test_put_meter(self, post_mock): + put_dict = {'metric': 'bla', 'timestamp': '0', + 'value': 123, 'tags': {'some_tag': 'foo'}} + self.client.put_meter(put_dict) + post_mock.assert_called_once_with( + 'http://127.0.0.1:4242/api/put?details', data=json.dumps(put_dict)) + + @mock.patch.object(requests, 'post') + def test_define_retention(self, post_mock): + self.client.define_retention('foo', 12) + post_mock.assert_called_once_with( + 'http://127.0.0.1:4242/api/uid/tsmeta?tsuid=foo', + data='{"tsuid": "foo", "retention": 12}') + + @mock.patch.object(requests, 'get') + def test_get_aggregators(self, get_mock): + self.client.get_aggregators() + get_mock.assert_called_once_with( + 'http://127.0.0.1:4242/api/aggregators') + + @mock.patch.object(requests, 'get') + def test_get_version(self, get_mock): + self.client.get_version() + get_mock.assert_called_once_with('http://127.0.0.1:4242/api/version') + + @mock.patch.object(requests, 'get') + def test_get_query(self, get_mock): + self.client.get_query('start=0&end=12&m=max:2-min:bla') + get_mock.assert_called_once_with( + 'http://127.0.0.1:4242/api/query?start=0&end=12&m=max:2-min:bla') diff --git a/opentsdbclient/utils.py b/opentsdbclient/utils.py new file mode 100644 index 0000000..77a5cb6 --- /dev/null +++ b/opentsdbclient/utils.py @@ -0,0 +1,23 @@ +# Copyright 2014: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +STATS_TEMPL = 'http://%(host)s:%(port)s/api/stats' +PUT_TEMPL = 'http://%(host)s:%(port)s/api/put?details' +META_TEMPL = 'http://%(host)s:%(port)s/api/uid/tsmeta?tsuid=%(tsuid)s' +CONF_TEMPL = 'http://%(host)s:%(oprt)s/api/config' +AGGR_TEMPL = 'http://%(host)s:%(port)s/api/aggregators' +VERSION_TEMPL = 'http://%(host)s:%(port)s/api/version' +QUERY_TEMPL = 'http://%(host)s:%(port)s/api/query?%(query)s' diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c79006a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +pbr>=0.6,!=0.7,<1.0 +requests>=1.1 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..877afa2 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,28 @@ +[metadata] +name = python-opentsdbclient +summary = Simple OpenTSDB REST Client Library +description-file = README.rst +author = Dina Belova +author-email = dbelova@mirantis.name +classifier = + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.3 + +[files] +packages = + opentsdbclient + +[global] +setup-hooks = + pbr.hooks.setup_hook + +[wheel] +universal = 1 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c584895 --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# Copyright 2014: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..8a4b66a --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,5 @@ +discover +hacking>=0.9.1,<0.10 +mock>=1.0 +testrepository>=0.0.18 +testtools>=0.9.34 \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..a79940e --- /dev/null +++ b/tox.ini @@ -0,0 +1,36 @@ +[tox] +envlist = py26,py27,py33,pep8 +minversion = 1.6 +skipsdist = True + +[testenv] +usedevelop = True +install_command = pip install -U {opts} {packages} +setenv = + VIRTUAL_ENV={envdir} + DISCOVER_DIRECTORY=opentsdbclient/tests + PYTHONHASHSEED=0 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + python setup.py testr --testr-args='{posargs}' + +[tox:jenkins] +downloadcache = ~/cache/pip + +[testenv:pep8] +commands = flake8 + +[testenv:cover] +commands = python setup.py testr --coverage --testr-args='{posargs}' + +[testenv:venv] +commands = {posargs} + +[testenv:docs] +commands= + python setup.py build_sphinx + +[flake8] +show-source = True +exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build \ No newline at end of file