metricd: start several processes to process more metric

This starts one process per CPU to maximize throughput in metric
treatment.

Change-Id: Iccc7e619cb9ed00bd010e66d1bdc009efe10b7f2

gnocchi/cli.py

@@ -17,6 +17,8 @@ try:
 except ImportError:
     import trollius as asyncio
 import logging
+import multiprocessing
+import time
 
 from gnocchi import indexer
 from gnocchi.indexer import sqlalchemy as sql_db
@@ -44,8 +46,9 @@ def statsd():
     statsd_service.start()
 
 
-def metricd():
-    conf = service.prepare_service()
+def _metricd(conf, cpu_number):
+    # Sleep a bit just not to start and poll everything at the same time.
+    time.sleep(cpu_number)
     s = storage.get_driver(conf)
     i = indexer.get_driver(conf)
     i.connect()
@@ -58,3 +61,21 @@ def metricd():
 
     process()
     loop.run_forever()
+
+
+def metricd():
+    conf = service.prepare_service()
+
+    def _wrap_metricd(cpu_number):
+        """Small wrapper for _metricd() that ensure it ALWAYS return.
+
+        Otherwise multiprocessing.Pool is stuck for ever.
+        """
+        try:
+            return _metricd(conf, cpu_number)
+        finally:
+            return
+    p = multiprocessing.Pool(conf.metricd.workers)
+    p.map(_wrap_metricd, range(conf.metricd.workers))
+    p.terminate()
+    p.join()

gnocchi/opts.py

@@ -29,6 +29,11 @@ import gnocchi.storage.swift
 def list_opts():
     return [
         ("indexer", gnocchi.indexer.OPTS),
+        ("metricd", (
+            cfg.Opt('workers', type=types.Integer(min=1),
+                    help='Number of workers for Gnocchi metric daemons. '
+                    'By default the available number of CPU is used.'),
+        )),
         ("api", (
             cfg.IntOpt('port',
                        default=8041,

gnocchi/service.py

@@ -55,6 +55,7 @@ def prepare_service(args=None):
         default_workers = 1
 
     conf.set_default("workers", default_workers, group="api")
+    conf.set_default("workers", default_workers, group="metricd")
 
     conf(args, project='gnocchi', validate_default_values=True)
     log.setup(conf, 'gnocchi')