diff --git a/grafana_dashboards/builder.py b/grafana_dashboards/builder.py index 1d30a4c..1534689 100644 --- a/grafana_dashboards/builder.py +++ b/grafana_dashboards/builder.py @@ -16,7 +16,6 @@ from oslo_config import cfg from grafana_dashboards.grafana import Grafana from grafana_dashboards.parser import YamlParser -from grafana_dashboards.schema.dashboard import Dashboard grafana_opts = [ cfg.StrOpt( @@ -42,7 +41,8 @@ class Builder(object): self.parser = YamlParser() def update_dashboard(self, path): - data = self.parser.load(path) - schema = Dashboard() - result = schema.validate(data) - self.grafana.create_dashboard(result, overwrite=True) + self.parser.parse(path) + dashboards = self.parser.data.get('dashboard', {}) + for item in dashboards: + data = self.parser.get_dashboard(item) + self.grafana.create_dashboard(data, overwrite=True) diff --git a/grafana_dashboards/grafana.py b/grafana_dashboards/grafana.py index 5b42a88..1ff6619 100644 --- a/grafana_dashboards/grafana.py +++ b/grafana_dashboards/grafana.py @@ -32,10 +32,13 @@ class Grafana(object): }) def create_dashboard(self, data, overwrite=False): - data['overwrite'] = overwrite + dashboard = { + 'dashboard': data, + 'overwrite': overwrite, + } headers = { 'Content-Type': 'application/json', } res = self.session.post( - self.url, data=json.dumps(data), headers=headers) + self.url, data=json.dumps(dashboard), headers=headers) res.raise_for_status() diff --git a/grafana_dashboards/parser.py b/grafana_dashboards/parser.py index 902c00b..45af31d 100644 --- a/grafana_dashboards/parser.py +++ b/grafana_dashboards/parser.py @@ -12,10 +12,41 @@ # License for the specific language governing permissions and limitations # under the License. +import io import yaml +from slugify import slugify + +from grafana_dashboards.schema.dashboard import Dashboard + class YamlParser(object): - def load(self, path): - return yaml.safe_load(open(path)) + def __init__(self): + self.data = {} + + def get_dashboard(self, slug): + return self.data.get('dashboard', {}).get(slug, None) + + def parse(self, fn): + with io.open(fn, 'r', encoding='utf-8') as fp: + self.parse_fp(fp) + + def parse_fp(self, fp): + data = yaml.safe_load(fp) + result = self.validate(data) + for item in result.items(): + group = self.data.get(item[0], {}) + # Create slug to make it easier to find dashboards. + title = item[1]['title'] + slug = slugify(title) + if slug in group: + raise Exception( + "Duplicate dashboard found in '{0}: '{1}' " + "already defined".format(fp.name, title)) + group[slug] = item[1] + self.data[item[0]] = group + + def validate(self, data): + schema = Dashboard() + return schema.validate(data) diff --git a/requirements.txt b/requirements.txt index 38952d0..0a85b9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,5 +4,6 @@ oslo.config>=1.11.0 oslo.log>=1.0.0,<1.1.0 +python-slugify PyYAML>=3.1.0 voluptuous>=0.7 diff --git a/tests/base.py b/tests/base.py index f6a8241..e7d2894 100644 --- a/tests/base.py +++ b/tests/base.py @@ -23,7 +23,6 @@ import re import testtools from grafana_dashboards.parser import YamlParser -from grafana_dashboards.schema import dashboard def get_scenarios(fixtures_path, in_ext='yaml', out_ext='json'): @@ -50,7 +49,6 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='json'): class TestCase(object): """Test case base class for all unit tests.""" - parser = YamlParser() def _read_raw_content(self): # if None assume empty file @@ -62,11 +60,11 @@ class TestCase(object): return content def test_yaml_snippet(self): + parser = YamlParser() expected_json = self._read_raw_content() - yaml_content = self.parser.load(self.in_filename) + parser.parse(self.in_filename) + valid_yaml = parser.data - schema = dashboard.Dashboard() - valid_yaml = schema.validate(yaml_content) pretty_json = json.dumps( valid_yaml, indent=4, separators=(',', ': '), sort_keys=True) diff --git a/tests/fixtures/parser/dashboard-0001.yaml b/tests/fixtures/parser/dashboard-0001.yaml new file mode 100644 index 0000000..2d47eec --- /dev/null +++ b/tests/fixtures/parser/dashboard-0001.yaml @@ -0,0 +1,2 @@ +dashboard: + title: New dashboard diff --git a/tests/fixtures/parser/dashboard-0002.yaml b/tests/fixtures/parser/dashboard-0002.yaml new file mode 100644 index 0000000..1589e70 --- /dev/null +++ b/tests/fixtures/parser/dashboard-0002.yaml @@ -0,0 +1,2 @@ +dashboard: + title: foobar diff --git a/tests/fixtures/parser/dashboard-0003.yaml b/tests/fixtures/parser/dashboard-0003.yaml new file mode 100644 index 0000000..6871c7a --- /dev/null +++ b/tests/fixtures/parser/dashboard-0003.yaml @@ -0,0 +1,5 @@ +dashboard: + title: New dashboard + rows: + - title: New row + height: 250px diff --git a/tests/schema/fixtures/dashboard-0001.json b/tests/schema/fixtures/dashboard-0001.json index ba680d7..cc98738 100644 --- a/tests/schema/fixtures/dashboard-0001.json +++ b/tests/schema/fixtures/dashboard-0001.json @@ -1,6 +1,8 @@ { "dashboard": { - "rows": [], - "title": "New dashboard" + "new-dashboard": { + "rows": [], + "title": "New dashboard" + } } } diff --git a/tests/schema/fixtures/dashboard-0002.json b/tests/schema/fixtures/dashboard-0002.json index 7140aab..67381e1 100644 --- a/tests/schema/fixtures/dashboard-0002.json +++ b/tests/schema/fixtures/dashboard-0002.json @@ -1,15 +1,17 @@ { "dashboard": { - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [], - "showTitle": false, - "title": "New row" - } - ], - "title": "New dashboard" + "new-dashboard": { + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [], + "showTitle": false, + "title": "New row" + } + ], + "title": "New dashboard" + } } } diff --git a/tests/schema/fixtures/dashboard-0003.json b/tests/schema/fixtures/dashboard-0003.json index 461be12..1f30291 100644 --- a/tests/schema/fixtures/dashboard-0003.json +++ b/tests/schema/fixtures/dashboard-0003.json @@ -1,27 +1,29 @@ { "dashboard": { - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "editable": true, - "error": false, - "limit": 10, - "mode": "starred", - "query": "", - "span": 12, - "tag": "", - "title": "Starred Dashboards", - "type": "dashlist" - } - ], - "showTitle": false, - "title": "New row" - } - ], - "title": "New dashboard" + "new-dashboard": { + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "editable": true, + "error": false, + "limit": 10, + "mode": "starred", + "query": "", + "span": 12, + "tag": "", + "title": "Starred Dashboards", + "type": "dashlist" + } + ], + "showTitle": false, + "title": "New row" + } + ], + "title": "New dashboard" + } } } diff --git a/tests/schema/fixtures/dashboard-0004.json b/tests/schema/fixtures/dashboard-0004.json index 61a9afb..c28f71d 100644 --- a/tests/schema/fixtures/dashboard-0004.json +++ b/tests/schema/fixtures/dashboard-0004.json @@ -1,25 +1,27 @@ { "dashboard": { - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "content": "Some example text is required.", - "editable": true, - "error": false, - "mode": "markdown", - "span": 12, - "title": "no title (click here)", - "type": "text" - } - ], - "showTitle": false, - "title": "New row" - } - ], - "title": "New dashboard" + "new-dashboard": { + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "content": "Some example text is required.", + "editable": true, + "error": false, + "mode": "markdown", + "span": 12, + "title": "no title (click here)", + "type": "text" + } + ], + "showTitle": false, + "title": "New row" + } + ], + "title": "New dashboard" + } } } diff --git a/tests/schema/fixtures/dashboard-0005.json b/tests/schema/fixtures/dashboard-0005.json index 0ad43c5..aa91cf3 100644 --- a/tests/schema/fixtures/dashboard-0005.json +++ b/tests/schema/fixtures/dashboard-0005.json @@ -1,35 +1,37 @@ { "dashboard": { - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "bars": false, - "editable": true, - "error": false, - "fill": 1, - "lines": true, - "linewidth": 2, - "percentage": false, - "pointradius": 5, - "points": false, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [], - "title": "no title (click here)", - "type": "graph", - "x-axis": true, - "y-axis": true - } - ], - "showTitle": false, - "title": "New row" - } - ], - "title": "New dashboard" + "new-dashboard": { + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "bars": false, + "editable": true, + "error": false, + "fill": 1, + "lines": true, + "linewidth": 2, + "percentage": false, + "pointradius": 5, + "points": false, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [], + "title": "no title (click here)", + "type": "graph", + "x-axis": true, + "y-axis": true + } + ], + "showTitle": false, + "title": "New row" + } + ], + "title": "New dashboard" + } } } diff --git a/tests/schema/fixtures/dashboard-0006.json b/tests/schema/fixtures/dashboard-0006.json index 82709bd..6b66f3e 100644 --- a/tests/schema/fixtures/dashboard-0006.json +++ b/tests/schema/fixtures/dashboard-0006.json @@ -1,35 +1,37 @@ { "dashboard": { - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "colorBackground": false, - "colorValue": false, - "editable": true, - "error": false, - "maxDataPoints": 100, - "span": 12, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [], - "thresholds": "", - "title": "no title (click here)", - "type": "singlestat", - "valueName": "avg" - } - ], - "showTitle": false, - "title": "New row" - } - ], - "title": "New dashboard" + "new-dashboard": { + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "colorBackground": false, + "colorValue": false, + "editable": true, + "error": false, + "maxDataPoints": 100, + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [], + "thresholds": "", + "title": "no title (click here)", + "type": "singlestat", + "valueName": "avg" + } + ], + "showTitle": false, + "title": "New row" + } + ], + "title": "New dashboard" + } } } diff --git a/tests/schema/fixtures/dashboard-0007.json b/tests/schema/fixtures/dashboard-0007.json index bff9364..824ec91 100644 --- a/tests/schema/fixtures/dashboard-0007.json +++ b/tests/schema/fixtures/dashboard-0007.json @@ -1,45 +1,47 @@ { "dashboard": { - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "editable": true, - "error": false, - "limit": 10, - "mode": "starred", - "query": "", - "span": 12, - "tag": "", - "title": "Starred Dashboards", - "type": "dashlist" - } - ], - "showTitle": false, - "title": "foo" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "content": "Some example text is required.", - "editable": true, - "error": false, - "mode": "markdown", - "span": 12, - "title": "no title (click here)", - "type": "text" - } - ], - "showTitle": false, - "title": "bar" - } - ], - "title": "New dashboard" + "new-dashboard": { + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "editable": true, + "error": false, + "limit": 10, + "mode": "starred", + "query": "", + "span": 12, + "tag": "", + "title": "Starred Dashboards", + "type": "dashlist" + } + ], + "showTitle": false, + "title": "foo" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "content": "Some example text is required.", + "editable": true, + "error": false, + "mode": "markdown", + "span": 12, + "title": "no title (click here)", + "type": "text" + } + ], + "showTitle": false, + "title": "bar" + } + ], + "title": "New dashboard" + } } } diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..ca0ddf0 --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,78 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from testtools import TestCase + +from grafana_dashboards import parser + + +class TestCaseParser(TestCase): + + def setUp(self): + super(TestCaseParser, self).setUp() + self.parser = parser.YamlParser() + + def test_get_dashboard_empty(self): + self._get_empty_dashboard('foobar') + + def test_parse_multiple(self): + path = os.path.join( + os.path.dirname(__file__), 'fixtures/parser/dashboard-0001.yaml') + self.parser.parse(path) + dashboard = { + 'foobar': {'rows': [], 'title': 'foobar'}, + 'new-dashboard': {'rows': [], 'title': 'New dashboard'}, + } + + # Get parsed dashboard + res = self.parser.get_dashboard('new-dashboard') + self.assertEqual(res, dashboard['new-dashboard']) + + # Check for a dashboard that does not exist + self._get_empty_dashboard('foobar') + + # Parse another file to ensure we are appending data. + path = os.path.join( + os.path.dirname(__file__), 'fixtures/parser/dashboard-0002.yaml') + self.parser.parse(path) + + res = self.parser.get_dashboard('foobar') + self.assertEqual(res, dashboard['foobar']) + + # Ensure our first dashboard still exists. + res = self.parser.get_dashboard('new-dashboard') + self.assertEqual(res, dashboard['new-dashboard']) + + def test_parse_duplicate(self): + path = os.path.join( + os.path.dirname(__file__), 'fixtures/parser/dashboard-0001.yaml') + self.parser.parse(path) + dashboard = { + 'new-dashboard': {'rows': [], 'title': 'New dashboard'}, + } + + # Get parsed dashboard + res = self.parser.get_dashboard('new-dashboard') + self.assertEqual(res, dashboard['new-dashboard']) + + path = os.path.join( + os.path.dirname(__file__), 'fixtures/parser/dashboard-0003.yaml') + # Fail to parse duplicate dashboard + self.assertRaises(Exception, self.parser.parse, path) + + def _get_empty_dashboard(self, name): + res = self.parser.get_dashboard(name) + self.assertEqual(res, None) diff --git a/tox.ini b/tox.ini index 92cd10c..81e6b7b 100644 --- a/tox.ini +++ b/tox.ini @@ -35,8 +35,9 @@ commands = oslo_debug_helper {posargs} [flake8] # E123, E125 skipped as they are invalid PEP-8. +# H202 skip until we actually write our own exceptions show-source = True -ignore = E123,E125 +ignore = E123,E125,H202 builtins = _ exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build