Adding collector service to distil

Add a new service called 'distil-collector', will be responsible for
usage collection.

Change-Id: Ib449c9cac217f0b98d7543918c9b8ca3541b0a10

distil/api/__init__.py

@@ -1,37 +0,0 @@
-# Copyright 2014 Catalyst IT Ltd
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-from oslo_config import cfg
-from oslo_log import log
-
-REST_SERVICE_OPTS = [
-    cfg.IntOpt('port',
-               default=9999,
-               help='The port for the Distil API server',
-               ),
-    cfg.StrOpt('host',
-               default='0.0.0.0',
-               help='The listen IP for the Distil API server',
-               ),
-    cfg.ListOpt('public_api_routes',
-                default=['/', '/v2/prices', '/v2/health'],
-                help='The list of public API routes',
-                ),
-]
-
-
-CONF = cfg.CONF
-CONF.register_opts(REST_SERVICE_OPTS)
-log.register_options(CONF)

distil/api/app.py

@@ -18,16 +18,13 @@ from oslo_config import cfg
 
 from distil.api import auth
 from distil.api import v2 as api_v2
-from distil.utils import api
 from distil import config
+from distil.utils import api
 
 CONF = cfg.CONF
 
 
 def make_app():
-    for group, opts in config.config_options():
-            CONF.register_opts(opts, group=group)
-
     app = flask.Flask(__name__)
 
     @app.route('/', methods=['GET'])

distil/cli/distil_api.py

@@ -41,8 +41,7 @@ class WritableLogger(object):
 
 
 def main():
-    CONF(project='distil', prog='distil-api')
-    log.setup(CONF, 'distil')
+    config.parse_args(sys.argv[1:], 'distil-api')
 
     application = app.make_app()
     CONF.log_opt_values(LOG, logging.INFO)

distil/cli/distil_collector.py

@@ -0,0 +1,34 @@
+# Copyright 2014 Catalyst IT Ltd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import sys
+
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_service import service
+
+from distil import config
+from distil.service import collector
+
+
+def main():
+    config.parse_args(sys.argv[1:], 'distil-collector')
+
+    srv = collector.CollectorService()
+    launcher = service.launch(cfg.CONF, srv)
+    launcher.wait()
+
+
+if __name__ == '__main__':
+    main()

distil/collector/ceilometer.py

@@ -0,0 +1,24 @@
+# Copyright 2016 Catalyst IT Ltd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+class CeilometerCollector(object):
+    def __init__(self, *arg, **kwargs):
+        pass

distil/config.py

@@ -15,11 +15,34 @@
 from oslo_config import cfg
 from oslo_log import log
 
+from distil import version
+
+CONF = cfg.CONF
+
 DEFAULT_OPTIONS = (
+    cfg.IntOpt('port',
+               default=9999,
+               help='The port for the Distil API server',
+               ),
+    cfg.StrOpt('host',
+               default='0.0.0.0',
+               help='The listen IP for the Distil API server',
+               ),
+    cfg.ListOpt('public_api_routes',
+                default=['/', '/v2/prices', '/v2/health'],
+                help='The list of public API routes',
+                ),
     cfg.ListOpt('ignore_tenants', default=[],
                 help=(''),),
 )
 
+COLLECTOR_OPTIONS = [
+    cfg.IntOpt('periodic_interval', default=3600,
+               help=('Interval of usage collection.')),
+    cfg.StrOpt('collector_backend', default='ceilometer',
+               help=('Data collector.')),
+]
+
 ODOO_OPTS = [
     cfg.StrOpt('version', default='8.0',
                help='Version of Odoo server.'),
@@ -38,11 +61,11 @@ ODOO_OPTS = [
 ]
 
 ODOO_GROUP = 'odoo'
+COLLECTOR_GROUP = 'collector'
 
-
-def config_options():
-    return [(None, DEFAULT_OPTIONS),
-            (ODOO_GROUP, ODOO_OPTS)]
+CONF.register_opts(DEFAULT_OPTIONS)
+CONF.register_opts(ODOO_OPTS, group=ODOO_GROUP)
+CONF.register_opts(COLLECTOR_OPTIONS, group=COLLECTOR_GROUP)
 
 # This is simply a namespace for global config storage
 main = None
@@ -72,3 +95,15 @@ def setup_config(conf):
     collection = conf['collection']
     global transformers
     transformers = conf['transformers']
+
+
+def parse_args(args=None, prog=None):
+    log.set_defaults()
+    log.register_options(CONF)
+    CONF(
+        args=args,
+        project='distil',
+        prog=prog,
+        version=version.version_info.version_string(),
+    )
+    log.setup(CONF, prog)

distil/service/collector.py

@@ -0,0 +1,61 @@
+# Copyright 2016 Catalyst IT Ltd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_service import service
+from oslo_service import threadgroup
+from stevedore import driver
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+class CollectorService(service.Service):
+    def __init__(self):
+        super(CollectorService, self).__init__()
+
+        self.thread_grp = None
+
+        collector_args = {}
+        self.collector = driver.DriverManager(
+            'distil.collector',
+            CONF.collector.collector_backend,
+            invoke_on_load=True,
+            invoke_kwds=collector_args).driver
+
+    def start(self):
+        LOG.info("Starting collector service...")
+
+        self.thread_grp = threadgroup.ThreadGroup()
+        self.thread_grp.add_timer(CONF.collector.periodic_interval,
+                                  self.collect_usage)
+
+        super(CollectorService, self).start()
+        LOG.info("Collector service started.")
+
+    def stop(self):
+        LOG.info("Stopping collector service gracefully...")
+
+        self.thread_grp.stop()
+        super(CollectorService, self).stop()
+
+        LOG.info("Collector service stoped.")
+
+    def reset(self):
+        super(CollectorService, self).reset()
+        logging.setup(CONF, 'distil-collector')
+
+    def collect_usage(self):
+        LOG.info("Begin to collect usage...")

setup.cfg

@@ -28,8 +28,12 @@ data_files =
 [entry_points]
 console_scripts =
     distil-api = distil.cli.distil_api:main
+    distil-collector = distil.cli.distil_collector:main
     distil-db-manage = distil.db.migration.cli:main
 
+distil.collector.backends =
+    ceilometer = distil.collector.ceilometer:CeilometerCollector
+
 distil.rater =
     file = distil.rater.file:FileRater
     odoo = distil.rater.odoo:OdooRater