sack-changer cli

support ability to change sacks via command line. will set new
sack value and clear any old sacks (if applicable). backlog must
be empty to change value.

Change-Id: Icf74b081e4cfaaaa607a5b9c684cbad4b8ecc006

doc/source/running.rst

@@ -162,6 +162,39 @@ If the estimated number of metrics is the absolute maximum, divide the value
 by 500 instead. If the estimated number of active metrics is conservative and
 expected to grow, divide the value by 100 instead to accommodate growth.
 
+How do we change sack size
+--------------------------
+
+In the event your system grows to capture signficantly more metrics than
+originally anticipated, the number of sacks can be changed to maintain good
+distribution. To avoid any loss of data when modifying `sacks` option. The
+option should be changed in the following order::
+
+  1. Stop all input services (api, statsd)
+
+  2. Stop all metricd services once backlog is cleared
+
+  3. Run gnocchi-change-sack-size <number of sacks> to set new sack size. Note
+     that sack value can only be changed if the backlog is empty.
+
+  4. Restart all gnocchi services (api, statsd, metricd) with new configuration
+
+Alternatively, to minimise API downtime::
+
+  1. Run gnocchi-upgrade but use a new incoming storage target such as a new
+     ceph pool, file path, etc... Additionally, set aggregate storage to a
+     new target as well.
+
+  2. Run gnocchi-change-sack-size <number of sacks> against new target
+
+  3. Stop all input services (api, statsd)
+
+  4. Restart all input services but target newly created incoming storage
+
+  5. When done clearing backlog from original incoming storage, switch all
+     metricd datemons to target new incoming storage but maintain original
+     aggregate storage.
+
 How to monitor Gnocchi
 ======================
 

gnocchi/cli.py

@@ -76,6 +76,26 @@ def upgrade():
         index.create_archive_policy_rule("default", "*", "low")
 
 
+def change_sack_size():
+    conf = cfg.ConfigOpts()
+    conf.register_cli_opts([
+        cfg.IntOpt("sack_size", required=True, min=1,
+                   help="Number of sacks."),
+    ])
+    conf = service.prepare_service(conf=conf)
+    s = storage.get_driver(conf)
+    report = s.incoming.measures_report(details=False)
+    remainder = report['summary']['measures']
+    if remainder:
+        LOG.error('Cannot change sack when non-empty backlog. Process '
+                  'remaining %s measures and try again', remainder)
+        return
+    LOG.info("Changing sack size to: %s", conf.sack_size)
+    old_num_sacks = s.incoming.get_storage_sacks()
+    s.incoming.set_storage_settings(conf.sack_size)
+    s.incoming.remove_sack_group(old_num_sacks)
+
+
 def statsd():
     statsd_service.start()
 

gnocchi/storage/incoming/_carbonara.py

@@ -67,6 +67,10 @@ class CarbonaraBasedStorage(incoming.StorageDriver):
     def set_storage_settings(num_sacks):
         raise NotImplementedError
 
+    @staticmethod
+    def remove_sack_group(num_sacks):
+        raise NotImplementedError
+
     def _unserialize_measures(self, measure_id, data):
         nb_measures = len(data) // self._MEASURE_SERIAL_LEN
         try:

gnocchi/storage/incoming/ceph.py

@@ -69,6 +69,14 @@ class CephStorage(_carbonara.CarbonaraBasedStorage):
         self.ioctx.write_full(self.CFG_PREFIX,
                               json.dumps({self.CFG_SACKS: num_sacks}).encode())
 
+    def remove_sack_group(self, num_sacks):
+        prefix = self.get_sack_prefix(num_sacks)
+        for i in six.moves.xrange(num_sacks):
+            try:
+                self.ioctx.remove_object(prefix % i)
+            except rados.ObjectNotFound:
+                pass
+
     def add_measures_batch(self, metrics_and_measures):
         names_by_sack = defaultdict(list)
         for metric, measures in six.iteritems(metrics_and_measures):

gnocchi/storage/incoming/file.py

@@ -16,6 +16,7 @@ import datetime
 import errno
 import json
 import os
+import shutil
 import tempfile
 import uuid
 
@@ -52,6 +53,11 @@ class FileStorage(_carbonara.CarbonaraBasedStorage):
         utils.ensure_paths([self._sack_path(i)
                             for i in six.moves.range(self.NUM_SACKS)])
 
+    def remove_sack_group(self, num_sacks):
+        prefix = self.get_sack_prefix(num_sacks)
+        for i in six.moves.xrange(num_sacks):
+            shutil.rmtree(os.path.join(self.basepath, prefix % i))
+
     def _sack_path(self, sack):
         return os.path.join(self.basepath, self.get_sack_name(sack))
 

gnocchi/storage/incoming/redis.py

@@ -34,6 +34,11 @@ class RedisStorage(_carbonara.CarbonaraBasedStorage):
     def set_storage_settings(self, num_sacks):
         self._client.hset(self.CFG_PREFIX, self.CFG_SACKS, num_sacks)
 
+    @staticmethod
+    def remove_sack_group(num_sacks):
+        # NOTE(gordc): redis doesn't maintain keys with empty values
+        pass
+
     def _build_measure_path(self, metric_id):
         return redis.SEP.join([
             self.get_sack_name(self.sack_for_metric(metric_id)),

gnocchi/storage/incoming/s3.py

@@ -59,6 +59,11 @@ class S3Storage(_carbonara.CarbonaraBasedStorage):
         # NOTE(gordc): override to follow s3 partitioning logic
         return '%s-' + ('%s/' % (num_sacks if num_sacks else self.NUM_SACKS))
 
+    @staticmethod
+    def remove_sack_group(num_sacks):
+        # nothing to cleanup since sacks are part of path
+        pass
+
     def upgrade(self, indexer, num_sacks):
         try:
             s3.create_bucket(self.s3, self._bucket_name_measures,

gnocchi/storage/incoming/swift.py

@@ -46,6 +46,11 @@ class SwiftStorage(_carbonara.CarbonaraBasedStorage):
         for i in six.moves.range(self.NUM_SACKS):
             self.swift.put_container(self.get_sack_name(i))
 
+    def remove_sack_group(self, num_sacks):
+        prefix = self.get_sack_prefix(num_sacks)
+        for i in six.moves.xrange(num_sacks):
+            self.swift.delete_container(prefix % i)
+
     def _store_new_measures(self, metric, data):
         now = datetime.datetime.utcnow().strftime("_%Y%m%d_%H:%M:%S")
         self.swift.put_object(

setup.cfg

@@ -133,6 +133,7 @@ gnocchi.rest.auth_helper =
 console_scripts =
     gnocchi-config-generator = gnocchi.cli:config_generator
     gnocchi-upgrade = gnocchi.cli:upgrade
+    gnocchi-change-sack-size = gnocchi.cli:change_sack_size
     gnocchi-statsd = gnocchi.cli:statsd
     gnocchi-metricd = gnocchi.cli:metricd