openstack
/
deb-gnocchi
Code Issues Proposed changes

swift: push to multiple sacks 

enable swift to push to multiple sacks and handle them accordingly

Change-Id: I0550961960805505d85919f30bfa1677dc390345
This commit is contained in:
gord chung 2017-04-05 15:04:21 +00:00
parent d5059fbef7
commit 85b41d699a
1 changed files with 33 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
gnocchi/storage/incoming/swift.py
View File

@ -30,61 +30,65 @@ class SwiftStorage(_carbonara.CarbonaraBasedStorage):
super(SwiftStorage, self).__init__(conf)
self.swift = swift.get_connection(conf)
def upgrade(self, indexer):
super(SwiftStorage, self).upgrade(indexer)
self.swift.put_container(self.MEASURE_PREFIX)
def upgrade(self, index):
super(SwiftStorage, self).upgrade(index)
for i in six.moves.range(self.NUM_SACKS):
self.swift.put_container(self.get_sack_name(i))
def _store_new_measures(self, metric, data):
now = datetime.datetime.utcnow().strftime("_%Y%m%d_%H:%M:%S")
self.swift.put_object(
self.MEASURE_PREFIX,
self.get_sack_name(self.sack_for_metric(metric.id)),
six.text_type(metric.id) + "/" + six.text_type(uuid.uuid4()) + now,
data)
def _build_report(self, details):
metric_details = defaultdict(int)
if details:
headers, files = self.swift.get_container(self.MEASURE_PREFIX,
full_listing=True)
for f in files:
metric, __ = f['name'].split("/", 1)
metric_details[metric] += 1
nb_metrics = len(metric_details)
else:
headers, files = self.swift.get_container(self.MEASURE_PREFIX,
delimiter='/',
full_listing=True)
nb_metrics = len(files)
measures = int(headers.get('x-container-object-count'))
return nb_metrics, measures, metric_details if details else None
nb_metrics = 0
measures = 0
for i in six.moves.range(self.NUM_SACKS):
if details:
headers, files = self.swift.get_container(
self.get_sack_name(i), full_listing=True)
for f in files:
metric, __ = f['name'].split("/", 1)
metric_details[metric] += 1
else:
headers, files = self.swift.get_container(
self.get_sack_name(i), delimiter='/', full_listing=True)
nb_metrics += len(files)
measures += int(headers.get('x-container-object-count'))
return (nb_metrics or len(metric_details), measures,
metric_details if details else None)
def list_metric_with_measures_to_process(self, sack):
headers, files = self.swift.get_container(self.MEASURE_PREFIX,
delimiter='/',
full_listing=True)
headers, files = self.swift.get_container(
self.get_sack_name(sack), delimiter='/', full_listing=True)
return set(f['subdir'][:-1] for f in files if 'subdir' in f)
def _list_measure_files_for_metric_id(self, metric_id):
def _list_measure_files_for_metric_id(self, sack, metric_id):
headers, files = self.swift.get_container(
self.MEASURE_PREFIX, path=six.text_type(metric_id),
self.get_sack_name(sack), path=six.text_type(metric_id),
full_listing=True)
return files
def delete_unprocessed_measures_for_metric_id(self, metric_id):
files = self._list_measure_files_for_metric_id(metric_id)
swift.bulk_delete(self.swift, self.MEASURE_PREFIX, files)
sack = self.sack_for_metric(metric_id)
files = self._list_measure_files_for_metric_id(sack, metric_id)
swift.bulk_delete(self.swift, self.get_sack_name(sack), files)
@contextlib.contextmanager
def process_measure_for_metric(self, metric):
files = self._list_measure_files_for_metric_id(metric.id)
sack = self.sack_for_metric(metric.id)
sack_name = self.get_sack_name(sack)
files = self._list_measure_files_for_metric_id(sack, metric.id)
measures = []
for f in files:
headers, data = self.swift.get_object(
self.MEASURE_PREFIX, f['name'])
headers, data = self.swift.get_object(sack_name, f['name'])
measures.extend(self._unserialize_measures(f['name'], data))
yield measures
# Now clean objects
swift.bulk_delete(self.swift, self.MEASURE_PREFIX, files)
swift.bulk_delete(self.swift, sack_name, files)