From 5b83fe64ccf8da8a9bbd98fe25005039f088b920 Mon Sep 17 00:00:00 2001 From: gord chung Date: Tue, 25 Apr 2017 21:03:05 +0000 Subject: [PATCH] sack-changer cli support ability to change sacks via command line. will set new sack value and clear any old sacks (if applicable). backlog must be empty to change value. Change-Id: Icf74b081e4cfaaaa607a5b9c684cbad4b8ecc006 --- doc/source/running.rst | 33 ++++++++++++++++++++++++++ gnocchi/cli.py | 20 ++++++++++++++++ gnocchi/storage/incoming/_carbonara.py | 4 ++++ gnocchi/storage/incoming/ceph.py | 8 +++++++ gnocchi/storage/incoming/file.py | 6 +++++ gnocchi/storage/incoming/redis.py | 5 ++++ gnocchi/storage/incoming/s3.py | 5 ++++ gnocchi/storage/incoming/swift.py | 5 ++++ setup.cfg | 1 + 9 files changed, 87 insertions(+) diff --git a/doc/source/running.rst b/doc/source/running.rst index f56f6541..09d901e7 100644 --- a/doc/source/running.rst +++ b/doc/source/running.rst @@ -162,6 +162,39 @@ If the estimated number of metrics is the absolute maximum, divide the value by 500 instead. If the estimated number of active metrics is conservative and expected to grow, divide the value by 100 instead to accommodate growth. +How do we change sack size +-------------------------- + +In the event your system grows to capture signficantly more metrics than +originally anticipated, the number of sacks can be changed to maintain good +distribution. To avoid any loss of data when modifying `sacks` option. The +option should be changed in the following order:: + + 1. Stop all input services (api, statsd) + + 2. Stop all metricd services once backlog is cleared + + 3. Run gnocchi-change-sack-size to set new sack size. Note + that sack value can only be changed if the backlog is empty. + + 4. Restart all gnocchi services (api, statsd, metricd) with new configuration + +Alternatively, to minimise API downtime:: + + 1. Run gnocchi-upgrade but use a new incoming storage target such as a new + ceph pool, file path, etc... Additionally, set aggregate storage to a + new target as well. + + 2. Run gnocchi-change-sack-size against new target + + 3. Stop all input services (api, statsd) + + 4. Restart all input services but target newly created incoming storage + + 5. When done clearing backlog from original incoming storage, switch all + metricd datemons to target new incoming storage but maintain original + aggregate storage. + How to monitor Gnocchi ====================== diff --git a/gnocchi/cli.py b/gnocchi/cli.py index ef47dcb9..7c3f6fd5 100644 --- a/gnocchi/cli.py +++ b/gnocchi/cli.py @@ -76,6 +76,26 @@ def upgrade(): index.create_archive_policy_rule("default", "*", "low") +def change_sack_size(): + conf = cfg.ConfigOpts() + conf.register_cli_opts([ + cfg.IntOpt("sack_size", required=True, min=1, + help="Number of sacks."), + ]) + conf = service.prepare_service(conf=conf) + s = storage.get_driver(conf) + report = s.incoming.measures_report(details=False) + remainder = report['summary']['measures'] + if remainder: + LOG.error('Cannot change sack when non-empty backlog. Process ' + 'remaining %s measures and try again', remainder) + return + LOG.info("Changing sack size to: %s", conf.sack_size) + old_num_sacks = s.incoming.get_storage_sacks() + s.incoming.set_storage_settings(conf.sack_size) + s.incoming.remove_sack_group(old_num_sacks) + + def statsd(): statsd_service.start() diff --git a/gnocchi/storage/incoming/_carbonara.py b/gnocchi/storage/incoming/_carbonara.py index f9a9f47a..2fe9bbd5 100644 --- a/gnocchi/storage/incoming/_carbonara.py +++ b/gnocchi/storage/incoming/_carbonara.py @@ -67,6 +67,10 @@ class CarbonaraBasedStorage(incoming.StorageDriver): def set_storage_settings(num_sacks): raise NotImplementedError + @staticmethod + def remove_sack_group(num_sacks): + raise NotImplementedError + def _unserialize_measures(self, measure_id, data): nb_measures = len(data) // self._MEASURE_SERIAL_LEN try: diff --git a/gnocchi/storage/incoming/ceph.py b/gnocchi/storage/incoming/ceph.py index e75fcfca..0f6c970a 100644 --- a/gnocchi/storage/incoming/ceph.py +++ b/gnocchi/storage/incoming/ceph.py @@ -69,6 +69,14 @@ class CephStorage(_carbonara.CarbonaraBasedStorage): self.ioctx.write_full(self.CFG_PREFIX, json.dumps({self.CFG_SACKS: num_sacks}).encode()) + def remove_sack_group(self, num_sacks): + prefix = self.get_sack_prefix(num_sacks) + for i in six.moves.xrange(num_sacks): + try: + self.ioctx.remove_object(prefix % i) + except rados.ObjectNotFound: + pass + def add_measures_batch(self, metrics_and_measures): names_by_sack = defaultdict(list) for metric, measures in six.iteritems(metrics_and_measures): diff --git a/gnocchi/storage/incoming/file.py b/gnocchi/storage/incoming/file.py index cd32e67f..070094e7 100644 --- a/gnocchi/storage/incoming/file.py +++ b/gnocchi/storage/incoming/file.py @@ -16,6 +16,7 @@ import datetime import errno import json import os +import shutil import tempfile import uuid @@ -52,6 +53,11 @@ class FileStorage(_carbonara.CarbonaraBasedStorage): utils.ensure_paths([self._sack_path(i) for i in six.moves.range(self.NUM_SACKS)]) + def remove_sack_group(self, num_sacks): + prefix = self.get_sack_prefix(num_sacks) + for i in six.moves.xrange(num_sacks): + shutil.rmtree(os.path.join(self.basepath, prefix % i)) + def _sack_path(self, sack): return os.path.join(self.basepath, self.get_sack_name(sack)) diff --git a/gnocchi/storage/incoming/redis.py b/gnocchi/storage/incoming/redis.py index 24c74716..fa5f1e88 100644 --- a/gnocchi/storage/incoming/redis.py +++ b/gnocchi/storage/incoming/redis.py @@ -34,6 +34,11 @@ class RedisStorage(_carbonara.CarbonaraBasedStorage): def set_storage_settings(self, num_sacks): self._client.hset(self.CFG_PREFIX, self.CFG_SACKS, num_sacks) + @staticmethod + def remove_sack_group(num_sacks): + # NOTE(gordc): redis doesn't maintain keys with empty values + pass + def _build_measure_path(self, metric_id): return redis.SEP.join([ self.get_sack_name(self.sack_for_metric(metric_id)), diff --git a/gnocchi/storage/incoming/s3.py b/gnocchi/storage/incoming/s3.py index 378de439..52016d11 100644 --- a/gnocchi/storage/incoming/s3.py +++ b/gnocchi/storage/incoming/s3.py @@ -59,6 +59,11 @@ class S3Storage(_carbonara.CarbonaraBasedStorage): # NOTE(gordc): override to follow s3 partitioning logic return '%s-' + ('%s/' % (num_sacks if num_sacks else self.NUM_SACKS)) + @staticmethod + def remove_sack_group(num_sacks): + # nothing to cleanup since sacks are part of path + pass + def upgrade(self, indexer, num_sacks): try: s3.create_bucket(self.s3, self._bucket_name_measures, diff --git a/gnocchi/storage/incoming/swift.py b/gnocchi/storage/incoming/swift.py index 5370771e..b0549d0f 100644 --- a/gnocchi/storage/incoming/swift.py +++ b/gnocchi/storage/incoming/swift.py @@ -46,6 +46,11 @@ class SwiftStorage(_carbonara.CarbonaraBasedStorage): for i in six.moves.range(self.NUM_SACKS): self.swift.put_container(self.get_sack_name(i)) + def remove_sack_group(self, num_sacks): + prefix = self.get_sack_prefix(num_sacks) + for i in six.moves.xrange(num_sacks): + self.swift.delete_container(prefix % i) + def _store_new_measures(self, metric, data): now = datetime.datetime.utcnow().strftime("_%Y%m%d_%H:%M:%S") self.swift.put_object( diff --git a/setup.cfg b/setup.cfg index 386d90f6..a2f544a7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -133,6 +133,7 @@ gnocchi.rest.auth_helper = console_scripts = gnocchi-config-generator = gnocchi.cli:config_generator gnocchi-upgrade = gnocchi.cli:upgrade + gnocchi-change-sack-size = gnocchi.cli:change_sack_size gnocchi-statsd = gnocchi.cli:statsd gnocchi-metricd = gnocchi.cli:metricd