Simple Kibana integration

* add simple proxy for kibana requests
* organize settings in `local_settings.py`
* narrow down coverage to monitoring package
* fix requirements with stable versions
* upgrade Horizon

Change-Id: I618485e9b6fa11fe423c3b1b3ad5f8c02cc163b4

.gitignore

@@ -1,6 +1,7 @@
 *.pyc
 *.swp
 *.sqlite3
+*.lock
 .environment_version
 .selenium_log
 .coverage*

ChangeLog

@@ -1,6 +1,43 @@
 CHANGES
 =======
 
+* Simple Kibana integration
+* Allow per-project grafana dashboards
+* Fix unittests
+* FIX the page title
+* added ability to filter all alarms with parameters in modal
+
+1.0.27
+------
+
+* fixed overview service/hostname bug
+* Added alarm definition pagination and notification
+* Add the ability to disable the notification panel
+* Provide default limit and offset for alarm_list
+* Wrapped the table cells to maintain alignment
+
+1.0.26
+------
+
+* Changed Edit Alarm to Edit Notification
+* Fixed metric selection in both Chrome and Firefox
+* Fix for retrieving alarms
+* Wrapped the metric chooser to handle long metrics
+* Added Alarm Id to the alarms tables
+* Added the SQL pagination for Alarms with offset and limit
+* Added pagination to the alarms table view
+* Response error message changed The error message was changed to make it more user friendly with a message that is more clear about the unsupported operation
+
+2015.1
+------
+
+* Modifying the README for grafana
+
+1.0.25
+------
+
+* Grabbing the list_measurements when getting metrics_list
+* alarams graph link broken
 * Allow dynamic dashboard links
 
 1.0.24

monitoring/api/monitor.py

@@ -14,9 +14,10 @@
 
 import logging
 
-from django.conf import settings  # noqa
 from monascaclient import client as monasca_client
 from openstack_dashboard.api import base
+from monitoring.config import local_settings as settings
+
 
 LOG = logging.getLogger(__name__)
 
@@ -30,8 +31,7 @@ def format_parameters(params):
 
 
 def monasca_endpoint(request):
-    service_type = getattr(settings, 'MONITORING_SERVICE_TYPE', 'monitoring')
-    endpoint = base.url_for(request, service_type)
+    endpoint = base.url_for(request, settings.MONITORING_SERVICE_TYPE)
     if endpoint.endswith('/'):
         endpoint = endpoint[:-1]
     return endpoint
@@ -39,15 +39,13 @@ def monasca_endpoint(request):
 
 def monascaclient(request, password=None):
     api_version = "2_0"
-    insecure = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', False)
-    cacert = getattr(settings, 'OPENSTACK_SSL_CACERT', None)
     endpoint = monasca_endpoint(request)
     LOG.debug('monascaclient connection created using token "%s" , url "%s"' %
               (request.user.token.id, endpoint))
     kwargs = {
         'token': request.user.token.id,
-        'insecure': insecure,
-        'ca_file': cacert,
+        'insecure': settings.OPENSTACK_SSL_NO_VERIFY,
+        'ca_file': settings.OPENSTACK_SSL_CACERT,
         'username': request.user.username,
         'password': password
         # 'timeout': args.timeout,

monitoring/config/local_settings.py

@@ -1,12 +1,22 @@
+from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
+
 # Services being monitored
-MONITORING_SERVICES = [
-    {'name': _('OpenStack Services'),
-     'groupBy': 'service'},
-    {'name': _('Servers'),
-     'groupBy': 'hostname'}
-]
+MONITORING_SERVICES = getattr(
+    settings,
+    'MONITORING_SERVICES',
+    [
+        {'name': _('OpenStack Services'),
+         'groupBy': 'service'},
+        {'name': _('Servers'),
+         'groupBy': 'hostname'}
+    ]
+)
+
+MONITORING_SERVICE_TYPE = getattr(
+    settings, 'MONITORING_SERVICE_TYPE', 'monitoring'
+)
 
 # Grafana button titles/file names (global across all projects):
 GRAFANA_LINKS = [
@@ -14,6 +24,9 @@ GRAFANA_LINKS = [
     {'title': 'Monasca Health', 'fileName': 'monasca.json'}
 ]
 
+DEFAULT_LINKS = GRAFANA_LINKS
+DASHBOARDS = getattr(settings, 'GRAFANA_LINKS', GRAFANA_LINKS)
+
 #
 # Per project grafana button titles/file names.  If in this form,
 # '*' will be applied to all projects not explicitly listed.
@@ -29,3 +42,9 @@ GRAFANA_LINKS = [
 #        {'title': 'OpenStack Dashboard', 'fileName': 'project.json'},
 #        {'title': 'Add New Dashboard', 'fileName': 'empty.json'}]}
 #]
+
+ENABLE_KIBANA_BUTTON = getattr(settings, 'ENABLE_KIBANA_BUTTON', False)
+KIBANA_HOST = getattr(settings, 'KIBANA_HOST', 'http://192.168.10.4:5601/')
+
+OPENSTACK_SSL_NO_VERIFY = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', False)
+OPENSTACK_SSL_CACERT = getattr(settings, 'OPENSTACK_SSL_CACERT', None)

monitoring/dashboard.py

@@ -15,11 +15,10 @@
 #    under the License.
 
 from django.utils.translation import ugettext_lazy as _
-from django.conf import settings  # noqa
+from monitoring.config import local_settings as settings
 
 import horizon
 
-service_type = getattr(settings, 'MONITORING_SERVICE_TYPE', 'monitoring')
 
 class Monitoring(horizon.Dashboard):
     name = _("Monitoring")
@@ -27,6 +26,6 @@ class Monitoring(horizon.Dashboard):
     panels = ('overview', 'alarmdefs', 'alarms', 'notifications',)
     default_panel = 'overview'
     policy_rules = (("monitoring", "monitoring:monitoring"),)
-    permissions = (('openstack.services.' + service_type),)
+    permissions = (('openstack.services.' + settings.MONITORING_SERVICE_TYPE),)
 
 horizon.register(Monitoring)

monitoring/overview/templates/overview/index.html

@@ -14,6 +14,12 @@
               {% trans dashboard.title %}
           </a>
       {% endfor %}
+      {% if can_access_logs and enable_kibana_button %}
+      <a target="dashboard" href="{% url 'horizon:monitoring:overview:kibana_proxy' url='/' %}" class="btn btn-default btn-sm">
+              <span class="glyphicon glyphicon-dashboard"></span>
+              Log Management
+      </a>
+      {% endif %}
     </div>
     {%  include 'monitoring/overview/monitor.html' %}
 {% endblock %}

monitoring/overview/tests.py

@@ -1,8 +1,11 @@
+# coding=utf-8
 from django.core import urlresolvers
+from django.test import RequestFactory
 from mock import patch, call  # noqa
 
 from monitoring.test import helpers
 from monitoring.overview import constants
+from monitoring.overview import views
 
 
 INDEX_URL = urlresolvers.reverse(
@@ -16,3 +19,32 @@ class OverviewTest(helpers.TestCase):
             res, 'monitoring/overview/index.html')
         self.assertTemplateUsed(res, 'monitoring/overview/monitor.html')
 
+
+class KibanaProxyViewTest(helpers.TestCase):
+
+    def setUp(self):
+        super(KibanaProxyViewTest, self).setUp()
+        self.view = views.KibanaProxyView()
+        self.request_factory = RequestFactory()
+
+    def test_get_relative_url_with_unicode(self):
+        """Tests if it properly converts multibyte characters"""
+        import urlparse
+
+        self.view.request = self.request_factory.get(
+            '/', data={'a': 1, 'b': 2}
+        )
+        expected_path = ('/elasticsearch/.kibana/search'
+                    '/New-Saved-Search%E3%81%82')
+        expected_qs = {'a': ['1'], 'b': ['2']}
+
+        url = self.view.get_relative_url(
+            u'/elasticsearch/.kibana/search/New-Saved-Searchあ'
+        )
+        # order of query params may change
+        parsed_url = urlparse.urlparse(url)
+        actual_path = parsed_url.path
+        actual_qs = urlparse.parse_qs(parsed_url.query)
+
+        self.assertEqual(actual_path, expected_path)
+        self.assertEqual(actual_qs, expected_qs)

monitoring/overview/urls.py

@@ -17,6 +17,8 @@ from django.conf.urls import patterns  # noqa
 from django.conf.urls import url  # noqa
 
 from monitoring.overview import views
+from monitoring.config import local_settings as settings
+
 
 urlpatterns = patterns(
     '',
@@ -24,4 +26,8 @@ urlpatterns = patterns(
     url(r'^status', views.StatusView.as_view(), name='status'),
     url(r'^proxy\/(?P<restpath>.*)$', views.MonascaProxyView.as_view()),
     url(r'^proxy', views.MonascaProxyView.as_view(), name='proxy'),
+    url(r'^logs_proxy(?P<url>.*)$',
+        views.KibanaProxyView.as_view(
+            base_url=settings.KIBANA_HOST), name='kibana_proxy'
+        ),
 )

monitoring/overview/views.py

@@ -17,33 +17,25 @@
 import json
 import logging
 import urllib
+import urllib2
 
-from django.conf import settings  # noqa
 from django.contrib import messages
 from django.core.urlresolvers import reverse_lazy
 from django.http import HttpResponse  # noqa
 from django.views.generic import TemplateView  # noqa
 from django.utils.translation import ugettext_lazy as _  # noqa
+from django import http
+from django.views.decorators.csrf import csrf_exempt
+from django.views import generic
+from openstack_dashboard import policy
 
-from monitoring.overview import constants
-from monitoring.alarms import tables as alarm_tables
 from monitoring import api
+from monitoring.alarms import tables as alarm_tables
+from monitoring.config import local_settings as settings
+from monitoring.overview import constants
+
 
 LOG = logging.getLogger(__name__)
-OVERVIEW = [
-    {'name': _('OpenStack Services'),
-     'groupBy': 'service'},
-    {'name': _('Servers'),
-     'groupBy': 'hostname'}
-]
-
-DEFAULT_LINKS = [
-    {'title': 'Dashboard', 'fileName': 'openstack.json'},
-    {'title': 'Monasca Health', 'fileName': 'monasca.json'}
-]
-
-SERVICES = getattr(settings, 'MONITORING_SERVICES', OVERVIEW)
-DASHBOARDS = getattr(settings, 'GRAFANA_LINKS', DEFAULT_LINKS)
 
 
 def get_icon(status):
@@ -86,7 +78,7 @@ def get_dashboard_links(request):
     #
     non_project_keys = {'fileName','title'}
     try:
-        for project_link in DASHBOARDS:
+        for project_link in settings.DASHBOARDS:
             key = project_link.keys()[0]
             value = project_link.values()[0]
             if key in non_project_keys:
@@ -94,7 +86,7 @@ def get_dashboard_links(request):
                 # we're not indexed by project, just return
                 # the whole list.
                 #
-                return DASHBOARDS
+                return settings.DASHBOARDS
             elif key == request.user.project_name:
                 #
                 # we match this project, return the project
@@ -108,7 +100,7 @@ def get_dashboard_links(request):
                 # match
                 #
                 return value
-        return DEFAULT_LINKS
+        return settings.DEFAULT_LINKS
     except Exception:
         LOG.warn("Failed to parse dashboard links by project, returning defaults.")
         pass
@@ -116,7 +108,7 @@ def get_dashboard_links(request):
     # Extra safety here -- should have got a match somewhere above,
     # but fall back to defaults.
     #
-    return DASHBOARDS
+    return settings.DASHBOARDS
 
 def show_by_dimension(data, dim_name):
     if 'dimensions' in data['metrics'][0]:
@@ -149,7 +141,7 @@ def generate_status(request):
         service = alarm_tables.get_service(a)
         service_alarms = alarms_by_service.setdefault(service, [])
         service_alarms.append(a)
-    for row in SERVICES:
+    for row in settings.MONITORING_SERVICES:
         row['name'] = unicode(row['name'])
         if 'groupBy' in row:
             alarms_by_group = {}
@@ -174,7 +166,7 @@ def generate_status(request):
                 service['class'] = get_status(service_alarms)
                 service['icon'] = get_icon(service['class'])
                 service['display'] = unicode(service['display'])
-    return SERVICES
+    return settings.MONITORING_SERVICES
 
 
 class IndexView(TemplateView):
@@ -186,6 +178,10 @@ class IndexView(TemplateView):
         api_root = self.request.build_absolute_uri(proxy_url_path)
         context["api"] = api_root
         context["dashboards"] = get_dashboard_links(self.request)
+        context['can_access_logs'] = policy.check(
+            (('identity', 'admin_required'), ), self.request
+        )
+        context['enable_kibana_button'] = settings.ENABLE_KIBANA_BUTTON
         return context
 
 
@@ -254,3 +250,75 @@ class StatusView(TemplateView):
 
         return HttpResponse(json.dumps(ret),
                             content_type='application/json')
+
+
+class _HttpMethodRequest(urllib2.Request):
+
+    def __init__(self, method, url, **kwargs):
+        urllib2.Request.__init__(self, url, **kwargs)
+        self.method = method
+
+    def get_method(self):
+        return self.method
+
+
+def proxy_stream_generator(response):
+    while True:
+        chunk = response.read(1000 * 1024)
+        if not chunk:
+            break
+        yield chunk
+
+
+class KibanaProxyView(generic.View):
+
+    base_url = None
+    http_method_names = ['GET', 'POST', 'PUT', 'DELETE', 'HEAD']
+
+    def read(self, method, url, data, headers):
+
+        proxy_request_url = self.get_absolute_url(url)
+        proxy_request = _HttpMethodRequest(
+            method, proxy_request_url, data=data, headers=headers
+        )
+        try:
+            response = urllib2.urlopen(proxy_request)
+
+        except urllib2.HTTPError as e:
+            return http.HttpResponse(
+                e.read(), status=e.code
+            )
+        except urllib2.URLError as e:
+            return http.HttpResponse(e.reason, 404)
+
+        else:
+            status = response.getcode()
+            return http.StreamingHttpResponse(
+                proxy_stream_generator(response),
+                status=status,
+                content_type=response.headers['content-type']
+            )
+
+    @csrf_exempt
+    def dispatch(self, request, url):
+
+        if not url:
+            url = '/'
+
+        if request.method not in self.http_method_names:
+            return http.HttpResponseNotAllowed(request.method)
+        headers = {
+            'X-Auth-Token': request.user.token.id
+        }
+        return self.read(request.method, url, request.body, headers)
+
+    def get_relative_url(self, url):
+        url = urllib.quote(url.encode('utf-8'))
+        params_str = self.request.GET.urlencode()
+
+        if params_str:
+            return '{0}?{1}'.format(url, params_str)
+        return url
+
+    def get_absolute_url(self, url):
+        return self.base_url + self.get_relative_url(url).lstrip('/')

monitoring/test/settings.py

@@ -53,8 +53,6 @@ INSTALLED_APPS = (
     'compressor',
     'horizon',
     'openstack_dashboard',
-    'openstack_dashboard.dashboards.project',
-    'openstack_dashboard.dashboards.admin',
     'monitoring',
     'openstack_dashboard.dashboards.settings',
 )
@@ -64,8 +62,8 @@ AUTHENTICATION_BACKENDS = ('openstack_auth.backend.KeystoneBackend',)
 SITE_BRANDING = 'OpenStack'
 
 HORIZON_CONFIG = {
-    'dashboards': ('project',),
-    'default_dashboard': 'project',
+    'dashboards': ('settings', 'monitoring',),
+    'default_dashboard': 'settings',
     "password_validator": {
         "regex": '^.{8,18}$',
         "help_text": _("Password must be between 8 and 18 characters.")
@@ -139,7 +137,7 @@ SECURITY_GROUP_RULES = {
 
 NOSE_ARGS = ['--nocapture',
              '--nologcapture',
-             '--cover-package=openstack_dashboard',
+             '--cover-package=monitoring',
              '--cover-inclusive',
              '--all-modules']
 

test-requirements.txt

@@ -2,7 +2,7 @@
 hacking>=0.9.2,<0.10
 
 # fix version problems
-cffi==0.9.2
+cffi>0.9
 oslo.i18n==1.7.0
 oslo.utils==1.9.0
 oslo.serialization==1.7.0
@@ -11,7 +11,7 @@ coverage>=3.6
 django-nose
 mock>=1.0
 funcsigs
-mox>=0.5.3
+mox3>=0.7.0
 nodeenv
 nose
 nose-exclude
@@ -22,4 +22,4 @@ selenium
 # Docs Requirements
 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
 oslosphinx
-http://tarballs.openstack.org/horizon/horizon-2015.1.0.tar.gz#egg=horizon
+http://tarballs.openstack.org/horizon/horizon-master.tar.gz#egg=horizon

tox.ini

@@ -5,7 +5,7 @@ skipsdist = True
 
 [testenv]
 usedevelop = True
-install_command = pip install {opts} {packages}
+install_command = pip install -U {opts} {packages}
 setenv = VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/test-requirements.txt
        -r{toxinidir}/requirements.txt