Added surveil-pack-upload cmd

Change-Id: I555faf6c870addd1bb1a4331bb4cccf6c40b061b

Dockerfile

@@ -13,7 +13,7 @@ RUN apt-get install -y python-pycurl
 RUN shinken --init
 
 ## modules
-RUN mkdir /var/lib/shinken/share
+#RUN mkdir /var/lib/shinken/share
 RUN shinken install webui
 RUN shinken install auth-cfg-password
 RUN pip install influxdb && shinken install mod-influxdb
@@ -27,6 +27,15 @@ RUN chmod u+s /usr/lib/nagios/plugins/check_icmp
 RUN chmod u+s /bin/ping
 RUN chmod u+s /bin/ping6
 
+## Packs
+ADD shinken-tools/packs /packs
+ADD shinken-tools/plugins /plugins
+
+# Download packs from savoirfairelinux/monitoring-tools
+RUN apt-get install -y subversion && \
+    svn checkout https://github.com/savoirfairelinux/monitoring-tools/trunk/packs/generic-host /packs/generic-host && \
+    apt-get remove -y subversion
+
 ## configuration
 RUN rm -rf /etc/shinken
 ADD tools/docker/etc/shinken /etc/shinken
@@ -42,10 +51,10 @@ ADD .git /surveil/.git
 ADD README.rst surveil/README.rst
 
 ## Install
-RUN apt-get install -y python3-pip
-RUN pip3 install -r /surveil/requirements.txt
+RUN apt-get install -y python3-pip python-dev
+RUN pip install -r /surveil/requirements.txt
 RUN apt-get install -y git
-RUN cd surveil && python3 setup.py install
+RUN cd surveil && python setup.py install
 
 ### Supervisor
 RUN apt-get -y install supervisor
@@ -57,4 +66,7 @@ EXPOSE 7767
 # Surveil
 EXPOSE 8080
 
-CMD surveil-init && /usr/bin/supervisord
+CMD surveil-init && \
+    surveil-pack-upload --mongo-url=mongo --mongo-port=27017 /packs/linux-keystone/ && \
+    surveil-pack-upload --mongo-url=mongo --mongo-port=27017 /packs/generic-host/ && \
+    /usr/bin/supervisord

fig.yml

@@ -9,6 +9,8 @@ surveil:
 
 mongo:
   image: mongo
+  ports:
+   - "27017:27017"
 
 influxdb:
   image: tutum/influxdb

requirements.txt

@@ -2,4 +2,5 @@ pecan>=0.5.0
 pymongo>=2.7.2
 wsme
 requests
-
+python-keystoneclient
+python-novaclient

setup.cfg

@@ -13,6 +13,7 @@ packages =
 console_scripts =
     surveil-api = surveil.cmd.api:main
     surveil-init = surveil.cmd.init:main
+    surveil-pack-upload = surveil.cmd.pack_upload:main
 
 [build_sphinx]
 source-dir = doc/source

shinken-tools/packs/linux-keystone/README.rst

@@ -0,0 +1,4 @@
+linux-keystone
+==========
+
+Shinken pack for monitoring OpenStack Keystone

shinken-tools/packs/linux-keystone/debian/changelog

@@ -0,0 +1,5 @@
+monitoring-packs-sfl-linux-keystone (2015.1.2.11.16-1) unstable; urgency=low
+
+  * Initial release
+
+ -- Thibault Cohen <thibault.cohen@savoirfairelinux.com>  Tue, 05 Oct 2014 10:15:52 -0400

shinken-tools/packs/linux-keystone/debian/compat

@@ -0,0 +1 @@
+9

shinken-tools/packs/linux-keystone/debian/control

@@ -0,0 +1,15 @@
+Source: monitoring-packs-sfl-linux-keystone
+Section: python
+Priority: optional
+Maintainer: Alexandre Viau <alexandre.viau@savoirfairelinux.com>
+Build-Depends: debhelper (>= 8.0.0)
+Standards-Version: 3.9.4
+Homepage: https://github.com/savoirfairelinux/monitoring-tools
+Vcs-Browser: https://github.com/savoirfairelinux/monitoring-tools.git
+
+Package: monitoring-packs-sfl-linux-keystone
+Architecture: all
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Description: Shinken pack for monitoring OpenStack Keystone
+ More information is available on Github:
+ https://github.com/savoirfairelinux/sfl-shinken-packs

shinken-tools/packs/linux-keystone/debian/copyright

@@ -0,0 +1,29 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: monitoring-packs-sfl-linux-keystone
+Source: <https://github.com/savoirfairelinux/monitoring-tools>
+
+Files: *
+Copyright:  2015 Alexandre Viau <alexandre.viau@savoirfairelinux.com>
+License: GPL-3.0+
+
+License: GPL-3.0+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License version 3 can be found in "/usr/share/common-licenses/GPL-3".
+
+# Please also look if there are files or directories which have a
+# different copyright/license attached and list them here.
+# Please avoid to pick license terms that are more restrictive than the
+# packaged work, as it may make Debian's contributions unacceptable upstream.

shinken-tools/packs/linux-keystone/debian/install

@@ -0,0 +1,2 @@
+pack /etc/shinken/packs/linux-keystone
+package.json /etc/shinken/packs/linux-keystone

shinken-tools/packs/linux-keystone/debian/rules

@@ -0,0 +1,8 @@
+#!/usr/bin/make -f
+# -*- makefile -*-
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+
+%:
+	dh $@ 

shinken-tools/packs/linux-keystone/debian/source/format

@@ -0,0 +1 @@
+3.0 (quilt)

shinken-tools/packs/linux-keystone/doc/conf.py

@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+#
+import sys
+import os
+
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'source/linux-keystone'
+
+# General information about the project.
+project = u'linux-keystone'
+copyright = u'2015, Alexandre Viau <alexandre.viau@savoirfairelinux.com>'
+
+# The short X.Y version.
+version = ''
+# The full version, including alpha/beta/rc tags.
+release = '1'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['source/_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = False
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'linux-keystonedoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+  ('source/linux-keystone', 'linux-keystone.tex', u'linux-keystone Documentation',
+   u'Alexandre Viau', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('source/linux-keystone', 'linux-keystone', u'linux-keystone Documentation',
+     [u'Alexandre Viau'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('source/linux-keystone', 'linux-keystone', u'linux-keystone Documentation',
+   u'Alexandre Viau', 'linux-keystone', 'Shinken pack for monitoring OpenStack Keystone',
+   'Packs'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#texinfo_no_detailmenu = False

shinken-tools/packs/linux-keystone/doc/linux-keystone.rst

@@ -0,0 +1,48 @@
+linux-keystone
+==============
+
+Dependencies
+************
+
+Shinken Modules
+~~~~~~~~~~~~~~~
+
+This pack will create services which need the following modules :
+
+**List of Shinken modules used**
+
+Plugins
+~~~~~~~
+
+This pack will create services which need the following plugins :
+
+**List of plugins used**
+
+Network
+~~~~~~~
+
+This pack will create services which need the following protocol :
+
+**List of port/protocol used**
+
+Installation
+************
+
+Copy the pack folder in the packs folder defined in shinken.cfg (`cfg_dir=packs`)
+
+
+How to use it
+*************
+
+
+Settings
+~~~~~~~~
+
+This is the list of settings which can be redefined in the host definition
+
+**List of macros**
+
+Triggers
+~~~~~~~~
+
+**List of triggers**

shinken-tools/packs/linux-keystone/monitoring-packs-sfl-linux-keystone.spec

@@ -0,0 +1,54 @@
+#
+# Example spec file for cdplayer app...
+#
+%define raw_name    linux-keystone
+%define name        monitoring-packs-sfl-%{raw_name}
+%define version     2015.1.2.11.16
+%define release     1
+%define install_folder /usr/lib/
+
+Name:       %{name}
+Version:    %{version}
+Release:    %{release}%{?dist}
+License: GPL v3
+Summary: Shinken pack for monitoring OpenStack Keystone
+Group: Networking/Other
+Source: http://monitoring.savoirfairelinux.com/%{name}_%{version}.orig.tar.gz
+URL: http://monitoring.savoirfairelinux.com/
+Distribution: Savoir-faire Linux
+Vendor: Savoir-faire Linux
+Packager: Alexandre Viau <alexandre.viau@savoirfairelinux.com>
+BuildRoot:  %{_tmppath}/%{name}-%{version}
+BuildRequires: python-sphinx
+#Requires: python, python-dlnetsnmp
+
+%description 
+Shinken pack for monitoring OpenStack Keystone
+
+%prep
+%setup -q -n %{name}
+
+%install
+%{__rm} -rf %{buildroot}
+%{__install} -d -m 755 %{buildroot}/%{_sysconfdir}/shinken/packs/%{raw_name}
+%{__cp} -r pack/* %{buildroot}/%{_sysconfdir}/shinken/packs/%{raw_name}
+%{__install} -p -m 755 package.json %{buildroot}/%{_sysconfdir}/shinken/packs/%{raw_name}
+%{__install} -d -m 755 %{buildroot}/%{_docdir}/shinken/packs/%{raw_name}
+%{__cp} -r doc/* %{buildroot}/%{_docdir}/shinken/packs/%{raw_name}
+%{__rm} %{buildroot}/%{_docdir}/shinken/packs/%{raw_name}/conf.py
+%{__install} -d -m 755 %{buildroot}/%{_mandir}/man1/shinken/packs/%{raw_name}
+sphinx-build -b man -d doc/build/doctrees/source doc %{buildroot}/%{_mandir}/man1/shinken/packs/%{raw_name}
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files
+%docdir
+%{_docdir}/shinken/packs/%{raw_name}
+%{_mandir}/man1/shinken/packs/%{raw_name}
+%config
+%{_sysconfdir}/shinken/packs/
+
+%changelog
+* Fri Jan 02 2015 Alexandre Viau <alexandre.viau@savoirfairelinux.com>
+- Initial Release

shinken-tools/packs/linux-keystone/pack/commands.cfg

@@ -0,0 +1,10 @@
+# -----------------------------------------------------------------
+#
+#      linux-keystone Commands
+#
+# -----------------------------------------------------------------
+
+define command {
+                command_name                          CHECK_KEYSTONE
+                command_line                          $PLUGINSDIR$/check_keystone -H $HOSTADDRESS$ $OPTIONS$
+}

shinken-tools/packs/linux-keystone/pack/linux-keystone.pack

@@ -0,0 +1,21 @@
+{
+"name":"linux-keystone",
+
+"description":"Shinken pack for monitoring OpenStack Keystone",
+
+"path":"",
+
+"macros":{
+        "_AUTH_URL": {"type": "url",
+                     "description": "Auth URL"
+                    },
+        "_USERNAME": {"type": "string",
+                     "description": "OpenStack username"
+                    },
+        "_PASSWORD": {"type": "string",
+                     "description": "OpenStack password"
+                    },
+        "_TENANT": {"type": "string",
+                     "description": "OpenStack tenant name"
+                    },
+}

shinken-tools/packs/linux-keystone/pack/services/service1.cfg

@@ -0,0 +1,7 @@
+define service{
+   host_name	  	        linux-keystone
+   service_description      service1.cfg
+   use            	        linux-keystone-service
+   register       	        0
+   check_command            CHECK_KEYSTONE
+}

shinken-tools/packs/linux-keystone/pack/templates.cfg

@@ -0,0 +1,14 @@
+# Shinken pack for monitoring OpenStack Keystone
+define host{
+   name                 linux-keystone
+   use                  pack-generic-host
+   check_interval       1
+   register             0
+}
+
+define service{
+    use                             pack-generic-host-service
+    service-description             linux-keystone-service
+    name                            linux-keystone-service
+    register                        0
+}

shinken-tools/packs/linux-keystone/package.json

@@ -0,0 +1,23 @@
+{
+	"name": "linux-keystone",
+	"types": ["pack"],
+	"version": "1.0",
+	"homepage": "https://github.com/savoirfairelinux/monitoring-tools",
+	"author": "Alexandre Viau",
+    "description": "Shinken pack for monitoring OpenStack Keystone",
+	"contributors": [
+	  {
+		"name": "Alexandre Viau",
+		"email": "alexandre.viau@savoirfairelinux.com"
+	  }
+	],
+	"repository": "https://github.com/savoirfairelinux/monitoring-tools",
+	"keywords": [
+	  "pack",
+      ""
+	],
+	"dependencies": {
+	  "shinken": ">=1.4"
+	},
+	"license": "GPL"
+}

surveil/api/controllers/v1/datamodel/host.py

@@ -40,9 +40,13 @@ class Host(types.Base):
 
     notification_period = wsme.wsattr(wtypes.text, mandatory=True)
 
+    use = wsme.wsattr(wtypes.text, mandatory=False)
+    """The template to use for this host"""
+
     @classmethod
     def sample(cls):
         return cls(
+            use="generic-host",
             host_name="bogus-router",
             address="192.168.1.254",
             max_check_attempts=5,

surveil/cmd/pack_upload.py

@@ -0,0 +1,87 @@
+# Copyright 2014 - Savoir-Faire Linux inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Script to push a Shinken pack to Surveil"""
+
+import argparse
+import fnmatch
+import os
+import sys
+
+from pymongo import mongo_client
+from shinken.objects import config
+
+
+def main():
+    # Parse the arguments
+    parser = argparse.ArgumentParser(
+        prog='surveil-pack-upload',
+        add_help=False,
+    )
+    parser.add_argument('--mongo-url',
+                        default='localhost',
+                        help='Defaults to localhost', type=str)
+    parser.add_argument('--mongo-port',
+                        default=27017,
+                        help='Defaults to 27017', type=int)
+    parser.add_argument('pack', metavar='[Pack]', type=str, nargs=1,
+                        help='Pack directory')
+
+    (options, args) = parser.parse_known_args(sys.argv[1:])
+
+    pack_dir = options.pack[0]
+    pack_sub_dir = os.path.join(pack_dir, 'pack')
+    pack_name = os.path.basename(os.path.normpath(pack_dir))
+
+    # Find the .cfg files
+    cfg_files = [
+        os.path.join(dirpath, f)
+        for dirpath, dirnames, files in os.walk(pack_sub_dir)
+        for f in fnmatch.filter(files, '*.cfg')
+    ]
+
+    # Load the config
+    conf = config.Config()
+    loaded_conf = conf.read_config(cfg_files)
+    raw_objects = conf.read_config_buf(loaded_conf)
+
+    # Remove the empty items
+    non_empty_config = {k: v for k, v in raw_objects.items() if v}
+
+    for config_type in non_empty_config:
+        for config_item in non_empty_config[config_type]:
+            # Tag the config objects
+            config_item['SURVEIL_PACK_NAME'] = pack_name
+
+            # Replace lists with csv
+            items_to_modify = (
+                [i for i in config_item.items() if isinstance(i[1], list)]
+            )
+            for i in items_to_modify:
+                config_item[i[0]] = ','.join(i[1])
+
+    # Remove the existing pack from mongodb
+    mongo = mongo_client.MongoClient(host=options.mongo_url,
+                                     port=options.mongo_port)
+    mongo_shinken = mongo.shinken
+    for collection in (
+            mongo_shinken.collection_names(include_system_collections=False)
+    ):
+        mongo_shinken[collection].remove({'SURVEIL_PACK_NAME': pack_name})
+
+    # Add the replacement pack
+    for config_type in non_empty_config:
+            mongo_shinken[config_type + 's'].insert(
+                non_empty_config[config_type]
+            )

tox.ini

@@ -8,6 +8,7 @@ usedevelop = True
 install_command = pip install -U --force-reinstall {opts} {packages}
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
+       git+https://github.com/reazem/shinken.git@setup_py#egg=shinken
 
 commands = python setup.py testr --slowest --testr-args='{posargs}'
 
@@ -21,5 +22,5 @@ commands = {posargs}
 commands = python setup.py build_sphinx
 
 [flake8]
-exclude = .venv,.git,.tox,env,dist,*openstack/common*,*lib/python*/,*egg,build
+exclude = .venv,.git,.tox,env,dist,*openstack/common*,*lib/python*/,*egg,build,*doc/conf.py