diff --git a/doc/source/api.rst b/doc/source/api.rst index a6b7fad..7d0f52f 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -7,6 +7,10 @@ API Reference :members: :undoc-members: +.. automodule:: grafana_dashboards.grafana.datasource + :members: + :undoc-members: + .. automodule:: grafana_dashboards.grafana.dashboard :members: :undoc-members: diff --git a/grafana_dashboards/builder.py b/grafana_dashboards/builder.py index 35b9f79..3db6998 100644 --- a/grafana_dashboards/builder.py +++ b/grafana_dashboards/builder.py @@ -33,13 +33,14 @@ class Builder(object): key=config.get('grafana', 'apikey')) self.parser = YamlParser() - def delete_dashboard(self, path): + def delete(self, path): self.load_files(path) + datasources = self.parser.data.get('datasource', {}) + LOG.info('Number of datasources to be deleted: %d', len(datasources)) + self._delete_datasource(datasources) dashboards = self.parser.data.get('dashboard', {}) - for name in dashboards: - LOG.debug('Deleting grafana dashboard %s', name) - self.grafana.dashboard.delete(name) - self.cache.set(name, '') + LOG.info('Number of dashboards to be deleted: %d', len(dashboards)) + self._delete_dashboard(dashboards) def load_files(self, path): files_to_process = [] @@ -54,14 +55,49 @@ class Builder(object): for fn in files_to_process: self.parser.parse(fn) - def update_dashboard(self, path): + def update(self, path): self.load_files(path) + datasources = self.parser.data.get('datasource', {}) + LOG.info('Number of datasources to be updated: %d', len(datasources)) + self._update_datasource(datasources) dashboards = self.parser.data.get('dashboard', {}) - LOG.info('Number of dashboards generated: %d', len(dashboards)) - for name in dashboards: + LOG.info('Number of dashboards to be updated: %d', len(dashboards)) + self._update_dashboard(dashboards) + + def _delete_dashboard(self, data): + for name in data: + LOG.debug('Deleting grafana dashboard %s', name) + self.grafana.dashboard.delete(name) + self.cache.set(name, '') + + def _delete_datasource(self, data): + for name in data: + LOG.debug('Deleting grafana datasource %s', name) + datasource_id = self.grafana.datasource.is_datasource(name) + if datasource_id: + self.grafana.datasource.delete(datasource_id) + self.cache.set(name, '') + + def _update_dashboard(self, data): + for name in data: data, md5 = self.parser.get_dashboard(name) if self.cache.has_changed(name, md5): self.grafana.dashboard.create(name, data, overwrite=True) self.cache.set(name, md5) else: LOG.debug("'%s' has not changed" % name) + + def _update_datasource(self, data): + for name in data: + data, md5 = self.parser.get_datasource(name) + if self.cache.has_changed(name, md5): + # Check for existing datasource so we can find the + # datasource_id. + datasource_id = self.grafana.datasource.is_datasource(name) + if datasource_id: + self.grafana.datasource.update(datasource_id, data) + else: + self.grafana.datasource.create(name, data) + self.cache.set(name, md5) + else: + LOG.debug("'%s' has not changed" % name) diff --git a/grafana_dashboards/cmd.py b/grafana_dashboards/cmd.py index 232bf95..9d5f9c2 100644 --- a/grafana_dashboards/cmd.py +++ b/grafana_dashboards/cmd.py @@ -27,9 +27,9 @@ LOG = logging.getLogger(__name__) class Client(object): def delete(self): - LOG.info('Deleting dashboards in %s', self.args.path) + LOG.info('Deleting schema in %s', self.args.path) builder = Builder(self.config) - builder.delete_dashboard(self.args.path) + builder.delete(self.args.path) def main(self): self.parse_arguments() @@ -90,12 +90,12 @@ class Client(object): logging.basicConfig(level=logging.INFO) def update(self): - LOG.info('Updating dashboards in %s', self.args.path) + LOG.info('Updating schema in %s', self.args.path) builder = Builder(self.config) - builder.update_dashboard(self.args.path) + builder.update(self.args.path) def validate(self): - LOG.info('Validating dashboards in %s', self.args.path) + LOG.info('Validating schema in %s', self.args.path) # NOTE(pabelanger): Disable caching support by default, in an effort # to improve performance. self.config.set('cache', 'enabled', 'false') diff --git a/grafana_dashboards/grafana/__init__.py b/grafana_dashboards/grafana/__init__.py index 744a968..afac7e5 100644 --- a/grafana_dashboards/grafana/__init__.py +++ b/grafana_dashboards/grafana/__init__.py @@ -12,14 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -try: - from urllib.parse import urljoin -except ImportError: - from urlparse import urljoin - import requests from grafana_dashboards.grafana.dashboard import Dashboard +from grafana_dashboards.grafana.datasource import Datasource class Grafana(object): @@ -36,7 +32,6 @@ class Grafana(object): self.server = url self.auth = None - base_url = urljoin(self.server, 'api/dashboards/db/') session = requests.Session() session.headers.update({ 'Content-Type': 'application/json', @@ -47,4 +42,5 @@ class Grafana(object): self.auth = {'Authorization': 'Bearer %s' % key} session.headers.update(self.auth) - self.dashboard = Dashboard(base_url, session) + self.dashboard = Dashboard(self.server, session) + self.datasource = Datasource(self.server, session) diff --git a/grafana_dashboards/grafana/dashboard.py b/grafana_dashboards/grafana/dashboard.py index 68ca57f..3f634b6 100644 --- a/grafana_dashboards/grafana/dashboard.py +++ b/grafana_dashboards/grafana/dashboard.py @@ -14,18 +14,15 @@ import json -try: - from urllib.parse import urljoin -except ImportError: - from urlparse import urljoin - from requests import exceptions +from grafana_dashboards.grafana import utils + class Dashboard(object): def __init__(self, url, session): - self.url = url + self.url = utils.urljoin(url, 'api/dashboards/db/') self.session = session def create(self, name, data, overwrite=False): @@ -64,7 +61,7 @@ class Dashboard(object): :raises Exception: if dashboard failed to delete """ - url = urljoin(self.url, name) + url = utils.urljoin(self.url, name) self.session.delete(url) if self.is_dashboard(name): raise Exception('dashboard[%s] failed to delete' % name) @@ -78,7 +75,7 @@ class Dashboard(object): :rtype: dict or None """ - url = urljoin(self.url, name) + url = utils.urljoin(self.url, name) try: res = self.session.get(url) res.raise_for_status() diff --git a/grafana_dashboards/grafana/datasource.py b/grafana_dashboards/grafana/datasource.py new file mode 100644 index 0000000..d6edcca --- /dev/null +++ b/grafana_dashboards/grafana/datasource.py @@ -0,0 +1,127 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +from requests import exceptions + +from grafana_dashboards.grafana import utils + + +class Datasource(object): + + def __init__(self, url, session): + self.url = utils.urljoin(url, 'api/datasources/') + self.session = session + + def create(self, name, data): + """Create a new datasource + + :param name: URL friendly title of the datasource + :type name: str + :param data: Datasource model + :type data: dict + + :raises Exception: if datasource already exists + + """ + if self.is_datasource(name): + raise Exception('datasource[%s] already exists' % name) + + res = self.session.post( + self.url, data=json.dumps(data)) + + res.raise_for_status() + return res.json() + + def delete(self, datasource_id): + """Delete a datasource + + :param datasource_id: Id number of datasource + :type datasource_id: int + + :raises Exception: if datasource failed to delete + + """ + url = utils.urljoin(self.url, str(datasource_id)) + self.session.delete(url) + if self.get(datasource_id): + raise Exception('datasource[%s] failed to delete' % datasource_id) + + def get(self, datasource_id): + """Get a datasource + + :param datasource_id: Id number of datasource + :type datasource_id: int + + :rtype: dict or None + + """ + url = utils.urljoin(self.url, str(datasource_id)) + try: + res = self.session.get(url) + res.raise_for_status() + except exceptions.HTTPError: + return None + + return res.json() + + def get_all(self): + """List all datasource + + :rtype: dict + + """ + res = self.session.get(self.url) + res.raise_for_status() + + return res.json() + + def is_datasource(self, name): + """Check if a datasource exists + + :param name: URL friendly title of the dashboard + :type name: str + + :returns: if datasource exists return id number. + :rtype: int + + """ + datasources = self.get_all() + for datasource in datasources: + if datasource['name'].lower() == name.lower(): + return datasource['id'] + return 0 + + def update(self, datasource_id, data): + """Update an existing datasource + + :param datasource_id: URL friendly title of the dashboard + :type datasource_id: int + :param data: Datasource model + :type data: dict + :param overwrite: Overwrite existing dashboard with newer version or + with the same dashboard title + :type overwrite: bool + + :raises Exception: if datasource already exists + + """ + url = utils.urljoin(self.url, str(datasource_id)) + + res = self.session.put( + url, data=json.dumps(data)) + + res.raise_for_status() + return res.json() diff --git a/grafana_dashboards/grafana/utils.py b/grafana_dashboards/grafana/utils.py new file mode 100644 index 0000000..4c715c8 --- /dev/null +++ b/grafana_dashboards/grafana/utils.py @@ -0,0 +1,18 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +try: + from urllib.parse import urljoin # noqa +except ImportError: + from urlparse import urljoin # noqa diff --git a/grafana_dashboards/parser.py b/grafana_dashboards/parser.py index 6f17df6..02ac110 100644 --- a/grafana_dashboards/parser.py +++ b/grafana_dashboards/parser.py @@ -20,7 +20,9 @@ import yaml from slugify import slugify -from grafana_dashboards.schema.dashboard import Dashboard +from grafana_dashboards.schema import Schema + +LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__) @@ -32,15 +34,18 @@ class YamlParser(object): def get_dashboard(self, slug): data = self.data.get('dashboard', {}).get(slug, None) - md5 = None - if data: - # Sort json keys to help our md5 hash are constant. - content = json.dumps(data, sort_keys=True) - md5 = hashlib.md5(content.encode('utf-8')).hexdigest() + md5 = self._generate_md5(data) LOG.debug('Dashboard %s: %s' % (slug, md5)) return data, md5 + def get_datasource(self, slug): + data = self.data.get('datasource', {}).get(slug, None) + md5 = self._generate_md5(data) + LOG.debug('Datasource %s: %s' % (slug, md5)) + + return data, md5 + def parse(self, fn): with io.open(fn, 'r', encoding='utf-8') as fp: self.parse_fp(fp) @@ -51,15 +56,26 @@ class YamlParser(object): for item in result.items(): group = self.data.get(item[0], {}) # Create slug to make it easier to find dashboards. - title = item[1]['title'] - slug = slugify(title) + if item[0] == 'dashboard': + name = item[1]['title'] + else: + name = item[1]['name'] + slug = slugify(name) if slug in group: raise Exception( - "Duplicate dashboard found in '{0}: '{1}' " - "already defined".format(fp.name, title)) + "Duplicate {0} found in '{1}: '{2}' " + "already defined".format(item[0], fp.name, name)) group[slug] = item[1] self.data[item[0]] = group def validate(self, data): - schema = Dashboard() + schema = Schema() return schema.validate(data) + + def _generate_md5(self, data): + md5 = None + if data: + # Sort json keys to help our md5 hash are constant. + content = json.dumps(data, sort_keys=True) + md5 = hashlib.md5(content.encode('utf-8')).hexdigest() + return md5 diff --git a/grafana_dashboards/schema/__init__.py b/grafana_dashboards/schema/__init__.py index e69de29..c718cec 100644 --- a/grafana_dashboards/schema/__init__.py +++ b/grafana_dashboards/schema/__init__.py @@ -0,0 +1,32 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import voluptuous as v + +from grafana_dashboards.schema.dashboard import Dashboard +from grafana_dashboards.schema.datasource import Datasource + + +class Schema(object): + + def validate(self, data): + dashboard = Dashboard().get_schema() + datasource = Datasource().get_schema() + + schema = v.Schema({ + v.Optional('dashboard'): dashboard, + v.Optional('datasource'): datasource, + }) + + return schema(data) diff --git a/grafana_dashboards/schema/dashboard.py b/grafana_dashboards/schema/dashboard.py index 6572f43..977c248 100644 --- a/grafana_dashboards/schema/dashboard.py +++ b/grafana_dashboards/schema/dashboard.py @@ -19,15 +19,11 @@ from grafana_dashboards.schema.row import Row class Dashboard(object): - def validate(self, data): + def get_schema(self): dashboard = { v.Required('title'): v.All(str, v.Length(min=1)), v.Optional('id'): int, } rows = Row().get_schema() dashboard.update(rows.schema) - schema = v.Schema({ - v.Required('dashboard'): dashboard, - }) - - return schema(data) + return dashboard diff --git a/grafana_dashboards/schema/datasource.py b/grafana_dashboards/schema/datasource.py new file mode 100644 index 0000000..2a27f3b --- /dev/null +++ b/grafana_dashboards/schema/datasource.py @@ -0,0 +1,29 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import voluptuous as v + + +class Datasource(object): + + def get_schema(self): + datasource = { + v.Required('access', default='direct'): v.Any('direct', 'proxy'), + v.Required('isDefault', default=False): v.All(bool), + v.Required('name'): v.All(str, v.Length(min=1)), + v.Required('type', default='graphite'): v.Any('graphite'), + v.Required('url'): v.All(str, v.Length(min=1)), + v.Optional('orgId'): int, + } + return datasource diff --git a/tests/fixtures/builder/datasource-0001.yaml b/tests/fixtures/builder/datasource-0001.yaml new file mode 100644 index 0000000..2a773de --- /dev/null +++ b/tests/fixtures/builder/datasource-0001.yaml @@ -0,0 +1,3 @@ +datasource: + name: Default + url: http://graphite.example.org:8080 diff --git a/tests/fixtures/builder/datasource-0002.yaml b/tests/fixtures/builder/datasource-0002.yaml new file mode 100644 index 0000000..a1f706d --- /dev/null +++ b/tests/fixtures/builder/datasource-0002.yaml @@ -0,0 +1,3 @@ +datasource: + name: Default + url: http://graphite.example.net:8080 diff --git a/tests/grafana/__init__.py b/tests/grafana/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/grafana/test_datasource.py b/tests/grafana/test_datasource.py new file mode 100644 index 0000000..5b2f21c --- /dev/null +++ b/tests/grafana/test_datasource.py @@ -0,0 +1,99 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import requests_mock +from testtools import TestCase + +from grafana_dashboards.grafana import Grafana + +DATASOURCE001 = { + "id": 1, + "orgId": 1, + "name": "foobar", + "type": "graphite", + "access": "direct", + "url": "http://example.org:8080", + "password": "", + "user": "", + "database": "", + "basicAuth": False, + "basicAuthUser": "", + "basicAuthPassword": "", + "isDefault": True, + "jsonData": None, +} + +DATASOURCE_NOT_FOUND = { + "message": "Failed to query datasources" +} + + +class TestCaseDatasource(TestCase): + + def setUp(self): + super(TestCaseDatasource, self).setUp() + self.url = 'http://localhost' + self.grafana = Grafana(self.url) + + @requests_mock.Mocker() + def test_create_new(self, mock_requests): + mock_requests.post('/api/datasources/', json=DATASOURCE001) + mock_requests.get('/api/datasources/', json=[]) + res = self.grafana.datasource.create('foobar', DATASOURCE001) + self.assertEqual(res, DATASOURCE001) + + @requests_mock.Mocker() + def test_get_not_found(self, mock_requests): + mock_requests.get( + '/api/datasources/1', json=DATASOURCE_NOT_FOUND, + status_code=404) + res = self.grafana.datasource.get(1) + self.assertEqual(res, None) + + @requests_mock.Mocker() + def test_get_success(self, mock_requests): + mock_requests.get('/api/datasources/1', json=DATASOURCE001) + res = self.grafana.datasource.get(1) + self.assertEqual(res, DATASOURCE001) + + @requests_mock.Mocker() + def test_get_all(self, mock_requests): + mock_requests.get( + '/api/datasources/', json=[DATASOURCE001]) + res = self.grafana.datasource.get_all() + self.assertEqual(res, [DATASOURCE001]) + + @requests_mock.Mocker() + def test_get_all_empty(self, mock_requests): + mock_requests.get('/api/datasources/', json=[]) + res = self.grafana.datasource.get_all() + self.assertEqual(res, []) + + @requests_mock.Mocker() + def test_is_datasource_empty(self, mock_requests): + mock_requests.get('/api/datasources/', json=[]) + res = self.grafana.datasource.is_datasource('foobar') + self.assertFalse(res) + + @requests_mock.Mocker() + def test_is_datasource_false(self, mock_requests): + mock_requests.get('/api/datasources/', json=[DATASOURCE001]) + res = self.grafana.datasource.is_datasource('new') + self.assertFalse(res) + + @requests_mock.Mocker() + def test_is_datasource_true(self, mock_requests): + mock_requests.get('/api/datasources/', json=[DATASOURCE001]) + res = self.grafana.datasource.is_datasource('foobar') + self.assertTrue(res) diff --git a/tests/schema/fixtures/datasource-0001.json b/tests/schema/fixtures/datasource-0001.json new file mode 100644 index 0000000..26f402c --- /dev/null +++ b/tests/schema/fixtures/datasource-0001.json @@ -0,0 +1,11 @@ +{ + "datasource": { + "new-datasource": { + "access": "direct", + "isDefault": false, + "name": "New datasource", + "type": "graphite", + "url": "http://example.org" + } + } +} diff --git a/tests/schema/fixtures/datasource-0001.yaml b/tests/schema/fixtures/datasource-0001.yaml new file mode 100644 index 0000000..77c9dfc --- /dev/null +++ b/tests/schema/fixtures/datasource-0001.yaml @@ -0,0 +1,3 @@ +datasource: + name: New datasource + url: http://example.org diff --git a/tests/schema/test_dashboard.py b/tests/schema/test_schema.py similarity index 92% rename from tests/schema/test_dashboard.py rename to tests/schema/test_schema.py index 76c9172..2c24fc0 100644 --- a/tests/schema/test_dashboard.py +++ b/tests/schema/test_schema.py @@ -21,6 +21,6 @@ from tests.base import get_scenarios from tests.schema.base import TestCase as BaseTestCase -class TestCaseSchemaDashboard(TestWithScenarios, TestCase, BaseTestCase): +class TestCaseSchema(TestWithScenarios, TestCase, BaseTestCase): fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures') scenarios = get_scenarios(fixtures_path) diff --git a/tests/test_builder.py b/tests/test_builder.py index 43366e6..4680e12 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -36,7 +36,7 @@ class TestCaseBuilder(TestCase): # Create a new builder to avoid duplicate dashboards. builder2 = builder.Builder(self.config) # Delete same dashboard, ensure we delete it from grafana. - builder2.delete_dashboard(path) + builder2.delete(path) self.assertEqual(mock_grafana.call_count, 1) def test_grafana_defaults(self): @@ -54,11 +54,56 @@ class TestCaseBuilder(TestCase): # Create a new builder to avoid duplicate dashboards. builder2 = builder.Builder(self.config) # Update again with same dashboard, ensure we don't update grafana. - builder2.update_dashboard(path) + builder2.update(path) self.assertEqual(mock_grafana.call_count, 0) + @mock.patch('grafana_dashboards.grafana.Datasource.create') + def test_create_datasource(self, mock_grafana): + path = os.path.join( + os.path.dirname(__file__), 'fixtures/builder/datasource-0001.yaml') + + # Create a datasource. + self._create_datasource(path) + # Create a new builder to avoid duplicate datasources. + builder2 = builder.Builder(self.config) + # Update again with same datasource, ensure we don't update grafana. + builder2.update(path) + self.assertEqual(mock_grafana.call_count, 0) + + @mock.patch( + 'grafana_dashboards.grafana.Datasource.is_datasource', + return_value=True) + @mock.patch('grafana_dashboards.grafana.Datasource.update') + def test_update_datasource(self, mock_is_datasource, mock_update): + path = os.path.join( + os.path.dirname(__file__), 'fixtures/builder/datasource-0001.yaml') + + # Create a datasource. + self._create_datasource(path) + # Create a new builder to avoid duplicate datasources. + builder2 = builder.Builder(self.config) + + # Same datasource name, different content. + path = os.path.join( + os.path.dirname(__file__), 'fixtures/builder/datasource-0002.yaml') + + # Update again with same datasource, ensure we update grafana. + builder2.update(path) + self.assertEqual(mock_is_datasource.call_count, 1) + self.assertEqual(mock_update.call_count, 1) + @mock.patch('grafana_dashboards.grafana.Dashboard.create') - def _update_dashboard(self, path, mock_grafana): - self.builder.update_dashboard(path) + def _update_dashboard(self, path, mock_create): + self.builder.update(path) # Cache is empty, so we should update grafana. - self.assertEqual(mock_grafana.call_count, 1) + self.assertEqual(mock_create.call_count, 1) + + @mock.patch( + 'grafana_dashboards.grafana.Datasource.is_datasource', + return_value=False) + @mock.patch('grafana_dashboards.grafana.Datasource.create') + def _create_datasource(self, path, mock_is_datasource, mock_create): + self.builder.update(path) + # Cache is empty, so we should update grafana. + self.assertEqual(mock_is_datasource.call_count, 1) + self.assertEqual(mock_create.call_count, 1)