Add datasource support

We now support the ability to create a datasource using yaml files.

Change-Id: I1db38ac25bc309398924c15635ea5dee4eaf264c
Signed-off-by: Paul Belanger <pabelanger@redhat.com>
This commit is contained in:
Paul Belanger 2015-10-16 13:25:59 -04:00
parent 57e4e17f20
commit 2c92819451
19 changed files with 466 additions and 51 deletions

View File

@ -7,6 +7,10 @@ API Reference
:members:
:undoc-members:
.. automodule:: grafana_dashboards.grafana.datasource
:members:
:undoc-members:
.. automodule:: grafana_dashboards.grafana.dashboard
:members:
:undoc-members:

View File

@ -33,13 +33,14 @@ class Builder(object):
key=config.get('grafana', 'apikey'))
self.parser = YamlParser()
def delete_dashboard(self, path):
def delete(self, path):
self.load_files(path)
datasources = self.parser.data.get('datasource', {})
LOG.info('Number of datasources to be deleted: %d', len(datasources))
self._delete_datasource(datasources)
dashboards = self.parser.data.get('dashboard', {})
for name in dashboards:
LOG.debug('Deleting grafana dashboard %s', name)
self.grafana.dashboard.delete(name)
self.cache.set(name, '')
LOG.info('Number of dashboards to be deleted: %d', len(dashboards))
self._delete_dashboard(dashboards)
def load_files(self, path):
files_to_process = []
@ -54,14 +55,49 @@ class Builder(object):
for fn in files_to_process:
self.parser.parse(fn)
def update_dashboard(self, path):
def update(self, path):
self.load_files(path)
datasources = self.parser.data.get('datasource', {})
LOG.info('Number of datasources to be updated: %d', len(datasources))
self._update_datasource(datasources)
dashboards = self.parser.data.get('dashboard', {})
LOG.info('Number of dashboards generated: %d', len(dashboards))
for name in dashboards:
LOG.info('Number of dashboards to be updated: %d', len(dashboards))
self._update_dashboard(dashboards)
def _delete_dashboard(self, data):
for name in data:
LOG.debug('Deleting grafana dashboard %s', name)
self.grafana.dashboard.delete(name)
self.cache.set(name, '')
def _delete_datasource(self, data):
for name in data:
LOG.debug('Deleting grafana datasource %s', name)
datasource_id = self.grafana.datasource.is_datasource(name)
if datasource_id:
self.grafana.datasource.delete(datasource_id)
self.cache.set(name, '')
def _update_dashboard(self, data):
for name in data:
data, md5 = self.parser.get_dashboard(name)
if self.cache.has_changed(name, md5):
self.grafana.dashboard.create(name, data, overwrite=True)
self.cache.set(name, md5)
else:
LOG.debug("'%s' has not changed" % name)
def _update_datasource(self, data):
for name in data:
data, md5 = self.parser.get_datasource(name)
if self.cache.has_changed(name, md5):
# Check for existing datasource so we can find the
# datasource_id.
datasource_id = self.grafana.datasource.is_datasource(name)
if datasource_id:
self.grafana.datasource.update(datasource_id, data)
else:
self.grafana.datasource.create(name, data)
self.cache.set(name, md5)
else:
LOG.debug("'%s' has not changed" % name)

View File

@ -27,9 +27,9 @@ LOG = logging.getLogger(__name__)
class Client(object):
def delete(self):
LOG.info('Deleting dashboards in %s', self.args.path)
LOG.info('Deleting schema in %s', self.args.path)
builder = Builder(self.config)
builder.delete_dashboard(self.args.path)
builder.delete(self.args.path)
def main(self):
self.parse_arguments()
@ -90,12 +90,12 @@ class Client(object):
logging.basicConfig(level=logging.INFO)
def update(self):
LOG.info('Updating dashboards in %s', self.args.path)
LOG.info('Updating schema in %s', self.args.path)
builder = Builder(self.config)
builder.update_dashboard(self.args.path)
builder.update(self.args.path)
def validate(self):
LOG.info('Validating dashboards in %s', self.args.path)
LOG.info('Validating schema in %s', self.args.path)
# NOTE(pabelanger): Disable caching support by default, in an effort
# to improve performance.
self.config.set('cache', 'enabled', 'false')

View File

@ -12,14 +12,10 @@
# License for the specific language governing permissions and limitations
# under the License.
try:
from urllib.parse import urljoin
except ImportError:
from urlparse import urljoin
import requests
from grafana_dashboards.grafana.dashboard import Dashboard
from grafana_dashboards.grafana.datasource import Datasource
class Grafana(object):
@ -36,7 +32,6 @@ class Grafana(object):
self.server = url
self.auth = None
base_url = urljoin(self.server, 'api/dashboards/db/')
session = requests.Session()
session.headers.update({
'Content-Type': 'application/json',
@ -47,4 +42,5 @@ class Grafana(object):
self.auth = {'Authorization': 'Bearer %s' % key}
session.headers.update(self.auth)
self.dashboard = Dashboard(base_url, session)
self.dashboard = Dashboard(self.server, session)
self.datasource = Datasource(self.server, session)

View File

@ -14,18 +14,15 @@
import json
try:
from urllib.parse import urljoin
except ImportError:
from urlparse import urljoin
from requests import exceptions
from grafana_dashboards.grafana import utils
class Dashboard(object):
def __init__(self, url, session):
self.url = url
self.url = utils.urljoin(url, 'api/dashboards/db/')
self.session = session
def create(self, name, data, overwrite=False):
@ -64,7 +61,7 @@ class Dashboard(object):
:raises Exception: if dashboard failed to delete
"""
url = urljoin(self.url, name)
url = utils.urljoin(self.url, name)
self.session.delete(url)
if self.is_dashboard(name):
raise Exception('dashboard[%s] failed to delete' % name)
@ -78,7 +75,7 @@ class Dashboard(object):
:rtype: dict or None
"""
url = urljoin(self.url, name)
url = utils.urljoin(self.url, name)
try:
res = self.session.get(url)
res.raise_for_status()

View File

@ -0,0 +1,127 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
from requests import exceptions
from grafana_dashboards.grafana import utils
class Datasource(object):
def __init__(self, url, session):
self.url = utils.urljoin(url, 'api/datasources/')
self.session = session
def create(self, name, data):
"""Create a new datasource
:param name: URL friendly title of the datasource
:type name: str
:param data: Datasource model
:type data: dict
:raises Exception: if datasource already exists
"""
if self.is_datasource(name):
raise Exception('datasource[%s] already exists' % name)
res = self.session.post(
self.url, data=json.dumps(data))
res.raise_for_status()
return res.json()
def delete(self, datasource_id):
"""Delete a datasource
:param datasource_id: Id number of datasource
:type datasource_id: int
:raises Exception: if datasource failed to delete
"""
url = utils.urljoin(self.url, str(datasource_id))
self.session.delete(url)
if self.get(datasource_id):
raise Exception('datasource[%s] failed to delete' % datasource_id)
def get(self, datasource_id):
"""Get a datasource
:param datasource_id: Id number of datasource
:type datasource_id: int
:rtype: dict or None
"""
url = utils.urljoin(self.url, str(datasource_id))
try:
res = self.session.get(url)
res.raise_for_status()
except exceptions.HTTPError:
return None
return res.json()
def get_all(self):
"""List all datasource
:rtype: dict
"""
res = self.session.get(self.url)
res.raise_for_status()
return res.json()
def is_datasource(self, name):
"""Check if a datasource exists
:param name: URL friendly title of the dashboard
:type name: str
:returns: if datasource exists return id number.
:rtype: int
"""
datasources = self.get_all()
for datasource in datasources:
if datasource['name'].lower() == name.lower():
return datasource['id']
return 0
def update(self, datasource_id, data):
"""Update an existing datasource
:param datasource_id: URL friendly title of the dashboard
:type datasource_id: int
:param data: Datasource model
:type data: dict
:param overwrite: Overwrite existing dashboard with newer version or
with the same dashboard title
:type overwrite: bool
:raises Exception: if datasource already exists
"""
url = utils.urljoin(self.url, str(datasource_id))
res = self.session.put(
url, data=json.dumps(data))
res.raise_for_status()
return res.json()

View File

@ -0,0 +1,18 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
try:
from urllib.parse import urljoin # noqa
except ImportError:
from urlparse import urljoin # noqa

View File

@ -20,7 +20,9 @@ import yaml
from slugify import slugify
from grafana_dashboards.schema.dashboard import Dashboard
from grafana_dashboards.schema import Schema
LOG = logging.getLogger(__name__)
LOG = logging.getLogger(__name__)
@ -32,15 +34,18 @@ class YamlParser(object):
def get_dashboard(self, slug):
data = self.data.get('dashboard', {}).get(slug, None)
md5 = None
if data:
# Sort json keys to help our md5 hash are constant.
content = json.dumps(data, sort_keys=True)
md5 = hashlib.md5(content.encode('utf-8')).hexdigest()
md5 = self._generate_md5(data)
LOG.debug('Dashboard %s: %s' % (slug, md5))
return data, md5
def get_datasource(self, slug):
data = self.data.get('datasource', {}).get(slug, None)
md5 = self._generate_md5(data)
LOG.debug('Datasource %s: %s' % (slug, md5))
return data, md5
def parse(self, fn):
with io.open(fn, 'r', encoding='utf-8') as fp:
self.parse_fp(fp)
@ -51,15 +56,26 @@ class YamlParser(object):
for item in result.items():
group = self.data.get(item[0], {})
# Create slug to make it easier to find dashboards.
title = item[1]['title']
slug = slugify(title)
if item[0] == 'dashboard':
name = item[1]['title']
else:
name = item[1]['name']
slug = slugify(name)
if slug in group:
raise Exception(
"Duplicate dashboard found in '{0}: '{1}' "
"already defined".format(fp.name, title))
"Duplicate {0} found in '{1}: '{2}' "
"already defined".format(item[0], fp.name, name))
group[slug] = item[1]
self.data[item[0]] = group
def validate(self, data):
schema = Dashboard()
schema = Schema()
return schema.validate(data)
def _generate_md5(self, data):
md5 = None
if data:
# Sort json keys to help our md5 hash are constant.
content = json.dumps(data, sort_keys=True)
md5 = hashlib.md5(content.encode('utf-8')).hexdigest()
return md5

View File

@ -0,0 +1,32 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import voluptuous as v
from grafana_dashboards.schema.dashboard import Dashboard
from grafana_dashboards.schema.datasource import Datasource
class Schema(object):
def validate(self, data):
dashboard = Dashboard().get_schema()
datasource = Datasource().get_schema()
schema = v.Schema({
v.Optional('dashboard'): dashboard,
v.Optional('datasource'): datasource,
})
return schema(data)

View File

@ -19,15 +19,11 @@ from grafana_dashboards.schema.row import Row
class Dashboard(object):
def validate(self, data):
def get_schema(self):
dashboard = {
v.Required('title'): v.All(str, v.Length(min=1)),
v.Optional('id'): int,
}
rows = Row().get_schema()
dashboard.update(rows.schema)
schema = v.Schema({
v.Required('dashboard'): dashboard,
})
return schema(data)
return dashboard

View File

@ -0,0 +1,29 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import voluptuous as v
class Datasource(object):
def get_schema(self):
datasource = {
v.Required('access', default='direct'): v.Any('direct', 'proxy'),
v.Required('isDefault', default=False): v.All(bool),
v.Required('name'): v.All(str, v.Length(min=1)),
v.Required('type', default='graphite'): v.Any('graphite'),
v.Required('url'): v.All(str, v.Length(min=1)),
v.Optional('orgId'): int,
}
return datasource

View File

@ -0,0 +1,3 @@
datasource:
name: Default
url: http://graphite.example.org:8080

View File

@ -0,0 +1,3 @@
datasource:
name: Default
url: http://graphite.example.net:8080

View File

View File

@ -0,0 +1,99 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import requests_mock
from testtools import TestCase
from grafana_dashboards.grafana import Grafana
DATASOURCE001 = {
"id": 1,
"orgId": 1,
"name": "foobar",
"type": "graphite",
"access": "direct",
"url": "http://example.org:8080",
"password": "",
"user": "",
"database": "",
"basicAuth": False,
"basicAuthUser": "",
"basicAuthPassword": "",
"isDefault": True,
"jsonData": None,
}
DATASOURCE_NOT_FOUND = {
"message": "Failed to query datasources"
}
class TestCaseDatasource(TestCase):
def setUp(self):
super(TestCaseDatasource, self).setUp()
self.url = 'http://localhost'
self.grafana = Grafana(self.url)
@requests_mock.Mocker()
def test_create_new(self, mock_requests):
mock_requests.post('/api/datasources/', json=DATASOURCE001)
mock_requests.get('/api/datasources/', json=[])
res = self.grafana.datasource.create('foobar', DATASOURCE001)
self.assertEqual(res, DATASOURCE001)
@requests_mock.Mocker()
def test_get_not_found(self, mock_requests):
mock_requests.get(
'/api/datasources/1', json=DATASOURCE_NOT_FOUND,
status_code=404)
res = self.grafana.datasource.get(1)
self.assertEqual(res, None)
@requests_mock.Mocker()
def test_get_success(self, mock_requests):
mock_requests.get('/api/datasources/1', json=DATASOURCE001)
res = self.grafana.datasource.get(1)
self.assertEqual(res, DATASOURCE001)
@requests_mock.Mocker()
def test_get_all(self, mock_requests):
mock_requests.get(
'/api/datasources/', json=[DATASOURCE001])
res = self.grafana.datasource.get_all()
self.assertEqual(res, [DATASOURCE001])
@requests_mock.Mocker()
def test_get_all_empty(self, mock_requests):
mock_requests.get('/api/datasources/', json=[])
res = self.grafana.datasource.get_all()
self.assertEqual(res, [])
@requests_mock.Mocker()
def test_is_datasource_empty(self, mock_requests):
mock_requests.get('/api/datasources/', json=[])
res = self.grafana.datasource.is_datasource('foobar')
self.assertFalse(res)
@requests_mock.Mocker()
def test_is_datasource_false(self, mock_requests):
mock_requests.get('/api/datasources/', json=[DATASOURCE001])
res = self.grafana.datasource.is_datasource('new')
self.assertFalse(res)
@requests_mock.Mocker()
def test_is_datasource_true(self, mock_requests):
mock_requests.get('/api/datasources/', json=[DATASOURCE001])
res = self.grafana.datasource.is_datasource('foobar')
self.assertTrue(res)

View File

@ -0,0 +1,11 @@
{
"datasource": {
"new-datasource": {
"access": "direct",
"isDefault": false,
"name": "New datasource",
"type": "graphite",
"url": "http://example.org"
}
}
}

View File

@ -0,0 +1,3 @@
datasource:
name: New datasource
url: http://example.org

View File

@ -21,6 +21,6 @@ from tests.base import get_scenarios
from tests.schema.base import TestCase as BaseTestCase
class TestCaseSchemaDashboard(TestWithScenarios, TestCase, BaseTestCase):
class TestCaseSchema(TestWithScenarios, TestCase, BaseTestCase):
fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
scenarios = get_scenarios(fixtures_path)

View File

@ -36,7 +36,7 @@ class TestCaseBuilder(TestCase):
# Create a new builder to avoid duplicate dashboards.
builder2 = builder.Builder(self.config)
# Delete same dashboard, ensure we delete it from grafana.
builder2.delete_dashboard(path)
builder2.delete(path)
self.assertEqual(mock_grafana.call_count, 1)
def test_grafana_defaults(self):
@ -54,11 +54,56 @@ class TestCaseBuilder(TestCase):
# Create a new builder to avoid duplicate dashboards.
builder2 = builder.Builder(self.config)
# Update again with same dashboard, ensure we don't update grafana.
builder2.update_dashboard(path)
builder2.update(path)
self.assertEqual(mock_grafana.call_count, 0)
@mock.patch('grafana_dashboards.grafana.Datasource.create')
def test_create_datasource(self, mock_grafana):
path = os.path.join(
os.path.dirname(__file__), 'fixtures/builder/datasource-0001.yaml')
# Create a datasource.
self._create_datasource(path)
# Create a new builder to avoid duplicate datasources.
builder2 = builder.Builder(self.config)
# Update again with same datasource, ensure we don't update grafana.
builder2.update(path)
self.assertEqual(mock_grafana.call_count, 0)
@mock.patch(
'grafana_dashboards.grafana.Datasource.is_datasource',
return_value=True)
@mock.patch('grafana_dashboards.grafana.Datasource.update')
def test_update_datasource(self, mock_is_datasource, mock_update):
path = os.path.join(
os.path.dirname(__file__), 'fixtures/builder/datasource-0001.yaml')
# Create a datasource.
self._create_datasource(path)
# Create a new builder to avoid duplicate datasources.
builder2 = builder.Builder(self.config)
# Same datasource name, different content.
path = os.path.join(
os.path.dirname(__file__), 'fixtures/builder/datasource-0002.yaml')
# Update again with same datasource, ensure we update grafana.
builder2.update(path)
self.assertEqual(mock_is_datasource.call_count, 1)
self.assertEqual(mock_update.call_count, 1)
@mock.patch('grafana_dashboards.grafana.Dashboard.create')
def _update_dashboard(self, path, mock_grafana):
self.builder.update_dashboard(path)
def _update_dashboard(self, path, mock_create):
self.builder.update(path)
# Cache is empty, so we should update grafana.
self.assertEqual(mock_grafana.call_count, 1)
self.assertEqual(mock_create.call_count, 1)
@mock.patch(
'grafana_dashboards.grafana.Datasource.is_datasource',
return_value=False)
@mock.patch('grafana_dashboards.grafana.Datasource.create')
def _create_datasource(self, path, mock_is_datasource, mock_create):
self.builder.update(path)
# Cache is empty, so we should update grafana.
self.assertEqual(mock_is_datasource.call_count, 1)
self.assertEqual(mock_create.call_count, 1)