From 42ef61dde03bd00b257ade8f60722c5f4c60ad85 Mon Sep 17 00:00:00 2001 From: Paul Belanger Date: Mon, 21 Sep 2015 16:41:21 -0400 Subject: [PATCH] Add a slug to better track dashboards Currently grafana will generate a slug based on the title name. For now, we can use an existing python library to do this. Ideally, we'll need to make sure the slugs generated between both are kept in sync. How hard can that be? Also move our validation logic into parser.py, since they are closely related. Change-Id: Id43f51d3f9db9131c25a8130f957572344a7d8f1 Signed-off-by: Paul Belanger --- grafana_dashboards/builder.py | 10 +-- grafana_dashboards/grafana.py | 7 +- grafana_dashboards/parser.py | 35 +++++++++- requirements.txt | 1 + tests/base.py | 8 +-- tests/fixtures/parser/dashboard-0001.yaml | 2 + tests/fixtures/parser/dashboard-0002.yaml | 2 + tests/fixtures/parser/dashboard-0003.yaml | 5 ++ tests/schema/fixtures/dashboard-0001.json | 6 +- tests/schema/fixtures/dashboard-0002.json | 24 ++++--- tests/schema/fixtures/dashboard-0003.json | 48 ++++++------- tests/schema/fixtures/dashboard-0004.json | 44 ++++++------ tests/schema/fixtures/dashboard-0005.json | 64 ++++++++--------- tests/schema/fixtures/dashboard-0006.json | 64 ++++++++--------- tests/schema/fixtures/dashboard-0007.json | 84 ++++++++++++----------- tests/test_parser.py | 78 +++++++++++++++++++++ tox.ini | 3 +- 17 files changed, 310 insertions(+), 175 deletions(-) create mode 100644 tests/fixtures/parser/dashboard-0001.yaml create mode 100644 tests/fixtures/parser/dashboard-0002.yaml create mode 100644 tests/fixtures/parser/dashboard-0003.yaml create mode 100644 tests/test_parser.py diff --git a/grafana_dashboards/builder.py b/grafana_dashboards/builder.py index 1d30a4c..1534689 100644 --- a/grafana_dashboards/builder.py +++ b/grafana_dashboards/builder.py @@ -16,7 +16,6 @@ from oslo_config import cfg from grafana_dashboards.grafana import Grafana from grafana_dashboards.parser import YamlParser -from grafana_dashboards.schema.dashboard import Dashboard grafana_opts = [ cfg.StrOpt( @@ -42,7 +41,8 @@ class Builder(object): self.parser = YamlParser() def update_dashboard(self, path): - data = self.parser.load(path) - schema = Dashboard() - result = schema.validate(data) - self.grafana.create_dashboard(result, overwrite=True) + self.parser.parse(path) + dashboards = self.parser.data.get('dashboard', {}) + for item in dashboards: + data = self.parser.get_dashboard(item) + self.grafana.create_dashboard(data, overwrite=True) diff --git a/grafana_dashboards/grafana.py b/grafana_dashboards/grafana.py index 5b42a88..1ff6619 100644 --- a/grafana_dashboards/grafana.py +++ b/grafana_dashboards/grafana.py @@ -32,10 +32,13 @@ class Grafana(object): }) def create_dashboard(self, data, overwrite=False): - data['overwrite'] = overwrite + dashboard = { + 'dashboard': data, + 'overwrite': overwrite, + } headers = { 'Content-Type': 'application/json', } res = self.session.post( - self.url, data=json.dumps(data), headers=headers) + self.url, data=json.dumps(dashboard), headers=headers) res.raise_for_status() diff --git a/grafana_dashboards/parser.py b/grafana_dashboards/parser.py index 902c00b..45af31d 100644 --- a/grafana_dashboards/parser.py +++ b/grafana_dashboards/parser.py @@ -12,10 +12,41 @@ # License for the specific language governing permissions and limitations # under the License. +import io import yaml +from slugify import slugify + +from grafana_dashboards.schema.dashboard import Dashboard + class YamlParser(object): - def load(self, path): - return yaml.safe_load(open(path)) + def __init__(self): + self.data = {} + + def get_dashboard(self, slug): + return self.data.get('dashboard', {}).get(slug, None) + + def parse(self, fn): + with io.open(fn, 'r', encoding='utf-8') as fp: + self.parse_fp(fp) + + def parse_fp(self, fp): + data = yaml.safe_load(fp) + result = self.validate(data) + for item in result.items(): + group = self.data.get(item[0], {}) + # Create slug to make it easier to find dashboards. + title = item[1]['title'] + slug = slugify(title) + if slug in group: + raise Exception( + "Duplicate dashboard found in '{0}: '{1}' " + "already defined".format(fp.name, title)) + group[slug] = item[1] + self.data[item[0]] = group + + def validate(self, data): + schema = Dashboard() + return schema.validate(data) diff --git a/requirements.txt b/requirements.txt index 38952d0..0a85b9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,5 +4,6 @@ oslo.config>=1.11.0 oslo.log>=1.0.0,<1.1.0 +python-slugify PyYAML>=3.1.0 voluptuous>=0.7 diff --git a/tests/base.py b/tests/base.py index f6a8241..e7d2894 100644 --- a/tests/base.py +++ b/tests/base.py @@ -23,7 +23,6 @@ import re import testtools from grafana_dashboards.parser import YamlParser -from grafana_dashboards.schema import dashboard def get_scenarios(fixtures_path, in_ext='yaml', out_ext='json'): @@ -50,7 +49,6 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='json'): class TestCase(object): """Test case base class for all unit tests.""" - parser = YamlParser() def _read_raw_content(self): # if None assume empty file @@ -62,11 +60,11 @@ class TestCase(object): return content def test_yaml_snippet(self): + parser = YamlParser() expected_json = self._read_raw_content() - yaml_content = self.parser.load(self.in_filename) + parser.parse(self.in_filename) + valid_yaml = parser.data - schema = dashboard.Dashboard() - valid_yaml = schema.validate(yaml_content) pretty_json = json.dumps( valid_yaml, indent=4, separators=(',', ': '), sort_keys=True) diff --git a/tests/fixtures/parser/dashboard-0001.yaml b/tests/fixtures/parser/dashboard-0001.yaml new file mode 100644 index 0000000..2d47eec --- /dev/null +++ b/tests/fixtures/parser/dashboard-0001.yaml @@ -0,0 +1,2 @@ +dashboard: + title: New dashboard diff --git a/tests/fixtures/parser/dashboard-0002.yaml b/tests/fixtures/parser/dashboard-0002.yaml new file mode 100644 index 0000000..1589e70 --- /dev/null +++ b/tests/fixtures/parser/dashboard-0002.yaml @@ -0,0 +1,2 @@ +dashboard: + title: foobar diff --git a/tests/fixtures/parser/dashboard-0003.yaml b/tests/fixtures/parser/dashboard-0003.yaml new file mode 100644 index 0000000..6871c7a --- /dev/null +++ b/tests/fixtures/parser/dashboard-0003.yaml @@ -0,0 +1,5 @@ +dashboard: + title: New dashboard + rows: + - title: New row + height: 250px diff --git a/tests/schema/fixtures/dashboard-0001.json b/tests/schema/fixtures/dashboard-0001.json index ba680d7..cc98738 100644 --- a/tests/schema/fixtures/dashboard-0001.json +++ b/tests/schema/fixtures/dashboard-0001.json @@ -1,6 +1,8 @@ { "dashboard": { - "rows": [], - "title": "New dashboard" + "new-dashboard": { + "rows": [], + "title": "New dashboard" + } } } diff --git a/tests/schema/fixtures/dashboard-0002.json b/tests/schema/fixtures/dashboard-0002.json index 7140aab..67381e1 100644 --- a/tests/schema/fixtures/dashboard-0002.json +++ b/tests/schema/fixtures/dashboard-0002.json @@ -1,15 +1,17 @@ { "dashboard": { - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [], - "showTitle": false, - "title": "New row" - } - ], - "title": "New dashboard" + "new-dashboard": { + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [], + "showTitle": false, + "title": "New row" + } + ], + "title": "New dashboard" + } } } diff --git a/tests/schema/fixtures/dashboard-0003.json b/tests/schema/fixtures/dashboard-0003.json index 461be12..1f30291 100644 --- a/tests/schema/fixtures/dashboard-0003.json +++ b/tests/schema/fixtures/dashboard-0003.json @@ -1,27 +1,29 @@ { "dashboard": { - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "editable": true, - "error": false, - "limit": 10, - "mode": "starred", - "query": "", - "span": 12, - "tag": "", - "title": "Starred Dashboards", - "type": "dashlist" - } - ], - "showTitle": false, - "title": "New row" - } - ], - "title": "New dashboard" + "new-dashboard": { + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "editable": true, + "error": false, + "limit": 10, + "mode": "starred", + "query": "", + "span": 12, + "tag": "", + "title": "Starred Dashboards", + "type": "dashlist" + } + ], + "showTitle": false, + "title": "New row" + } + ], + "title": "New dashboard" + } } } diff --git a/tests/schema/fixtures/dashboard-0004.json b/tests/schema/fixtures/dashboard-0004.json index 61a9afb..c28f71d 100644 --- a/tests/schema/fixtures/dashboard-0004.json +++ b/tests/schema/fixtures/dashboard-0004.json @@ -1,25 +1,27 @@ { "dashboard": { - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "content": "Some example text is required.", - "editable": true, - "error": false, - "mode": "markdown", - "span": 12, - "title": "no title (click here)", - "type": "text" - } - ], - "showTitle": false, - "title": "New row" - } - ], - "title": "New dashboard" + "new-dashboard": { + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "content": "Some example text is required.", + "editable": true, + "error": false, + "mode": "markdown", + "span": 12, + "title": "no title (click here)", + "type": "text" + } + ], + "showTitle": false, + "title": "New row" + } + ], + "title": "New dashboard" + } } } diff --git a/tests/schema/fixtures/dashboard-0005.json b/tests/schema/fixtures/dashboard-0005.json index 0ad43c5..aa91cf3 100644 --- a/tests/schema/fixtures/dashboard-0005.json +++ b/tests/schema/fixtures/dashboard-0005.json @@ -1,35 +1,37 @@ { "dashboard": { - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "bars": false, - "editable": true, - "error": false, - "fill": 1, - "lines": true, - "linewidth": 2, - "percentage": false, - "pointradius": 5, - "points": false, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [], - "title": "no title (click here)", - "type": "graph", - "x-axis": true, - "y-axis": true - } - ], - "showTitle": false, - "title": "New row" - } - ], - "title": "New dashboard" + "new-dashboard": { + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "bars": false, + "editable": true, + "error": false, + "fill": 1, + "lines": true, + "linewidth": 2, + "percentage": false, + "pointradius": 5, + "points": false, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [], + "title": "no title (click here)", + "type": "graph", + "x-axis": true, + "y-axis": true + } + ], + "showTitle": false, + "title": "New row" + } + ], + "title": "New dashboard" + } } } diff --git a/tests/schema/fixtures/dashboard-0006.json b/tests/schema/fixtures/dashboard-0006.json index 82709bd..6b66f3e 100644 --- a/tests/schema/fixtures/dashboard-0006.json +++ b/tests/schema/fixtures/dashboard-0006.json @@ -1,35 +1,37 @@ { "dashboard": { - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "colorBackground": false, - "colorValue": false, - "editable": true, - "error": false, - "maxDataPoints": 100, - "span": 12, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [], - "thresholds": "", - "title": "no title (click here)", - "type": "singlestat", - "valueName": "avg" - } - ], - "showTitle": false, - "title": "New row" - } - ], - "title": "New dashboard" + "new-dashboard": { + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "colorBackground": false, + "colorValue": false, + "editable": true, + "error": false, + "maxDataPoints": 100, + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [], + "thresholds": "", + "title": "no title (click here)", + "type": "singlestat", + "valueName": "avg" + } + ], + "showTitle": false, + "title": "New row" + } + ], + "title": "New dashboard" + } } } diff --git a/tests/schema/fixtures/dashboard-0007.json b/tests/schema/fixtures/dashboard-0007.json index bff9364..824ec91 100644 --- a/tests/schema/fixtures/dashboard-0007.json +++ b/tests/schema/fixtures/dashboard-0007.json @@ -1,45 +1,47 @@ { "dashboard": { - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "editable": true, - "error": false, - "limit": 10, - "mode": "starred", - "query": "", - "span": 12, - "tag": "", - "title": "Starred Dashboards", - "type": "dashlist" - } - ], - "showTitle": false, - "title": "foo" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "content": "Some example text is required.", - "editable": true, - "error": false, - "mode": "markdown", - "span": 12, - "title": "no title (click here)", - "type": "text" - } - ], - "showTitle": false, - "title": "bar" - } - ], - "title": "New dashboard" + "new-dashboard": { + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "editable": true, + "error": false, + "limit": 10, + "mode": "starred", + "query": "", + "span": 12, + "tag": "", + "title": "Starred Dashboards", + "type": "dashlist" + } + ], + "showTitle": false, + "title": "foo" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "content": "Some example text is required.", + "editable": true, + "error": false, + "mode": "markdown", + "span": 12, + "title": "no title (click here)", + "type": "text" + } + ], + "showTitle": false, + "title": "bar" + } + ], + "title": "New dashboard" + } } } diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..ca0ddf0 --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,78 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from testtools import TestCase + +from grafana_dashboards import parser + + +class TestCaseParser(TestCase): + + def setUp(self): + super(TestCaseParser, self).setUp() + self.parser = parser.YamlParser() + + def test_get_dashboard_empty(self): + self._get_empty_dashboard('foobar') + + def test_parse_multiple(self): + path = os.path.join( + os.path.dirname(__file__), 'fixtures/parser/dashboard-0001.yaml') + self.parser.parse(path) + dashboard = { + 'foobar': {'rows': [], 'title': 'foobar'}, + 'new-dashboard': {'rows': [], 'title': 'New dashboard'}, + } + + # Get parsed dashboard + res = self.parser.get_dashboard('new-dashboard') + self.assertEqual(res, dashboard['new-dashboard']) + + # Check for a dashboard that does not exist + self._get_empty_dashboard('foobar') + + # Parse another file to ensure we are appending data. + path = os.path.join( + os.path.dirname(__file__), 'fixtures/parser/dashboard-0002.yaml') + self.parser.parse(path) + + res = self.parser.get_dashboard('foobar') + self.assertEqual(res, dashboard['foobar']) + + # Ensure our first dashboard still exists. + res = self.parser.get_dashboard('new-dashboard') + self.assertEqual(res, dashboard['new-dashboard']) + + def test_parse_duplicate(self): + path = os.path.join( + os.path.dirname(__file__), 'fixtures/parser/dashboard-0001.yaml') + self.parser.parse(path) + dashboard = { + 'new-dashboard': {'rows': [], 'title': 'New dashboard'}, + } + + # Get parsed dashboard + res = self.parser.get_dashboard('new-dashboard') + self.assertEqual(res, dashboard['new-dashboard']) + + path = os.path.join( + os.path.dirname(__file__), 'fixtures/parser/dashboard-0003.yaml') + # Fail to parse duplicate dashboard + self.assertRaises(Exception, self.parser.parse, path) + + def _get_empty_dashboard(self, name): + res = self.parser.get_dashboard(name) + self.assertEqual(res, None) diff --git a/tox.ini b/tox.ini index 92cd10c..81e6b7b 100644 --- a/tox.ini +++ b/tox.ini @@ -35,8 +35,9 @@ commands = oslo_debug_helper {posargs} [flake8] # E123, E125 skipped as they are invalid PEP-8. +# H202 skip until we actually write our own exceptions show-source = True -ignore = E123,E125 +ignore = E123,E125,H202 builtins = _ exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build