openstack
/
gnocchi
Code Issues Proposed changes

tests: block when acquiring processing lock 

In tests, not blocking when processing the data in background might
trigger a race condition since all the tests run in parallel on the same
database.

Change-Id: Id808f9e659f242fdaecd52cad08a6d00c96ac051
This commit is contained in:
Julien Danjou 2015-11-24 12:07:01 +01:00
parent 54b7b70f3a
commit 9309cdc8f6
6 changed files with 35 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
gnocchi/storage/__init__.py
View File

@ -157,17 +157,31 @@ class StorageDriver(object):
def upgrade(index):
pass
def process_background_tasks(self, index):
def process_background_tasks(self, index, sync=False):
"""Process background tasks for this storage.
This calls :func:`process_measures` to process new measures and
:func:`expunge_metrics` to expunge deleted metrics.
:param index: An indexer to be used for querying metrics
:param sync: If True, then process everything synchronously and raise
on error
:type sync: bool
"""
LOG.debug("Processing new and to delete measures")
try:
self.process_measures(index)
self.process_measures(index, sync)
except Exception:
if sync:
raise
LOG.error("Unexpected error during measures processing",
exc_info=True)
LOG.debug("Expunging deleted metrics")
try:
self.expunge_metrics(index)
except Exception:
if sync:
raise
LOG.error("Unexpected error during deleting metrics",
exc_info=True)
@ -197,7 +211,7 @@ class StorageDriver(object):
raise exceptions.NotImplementedError
@staticmethod
def process_measures(indexer=None):
def process_measures(indexer=None, sync=False):
"""Process added measures in background.
Some drivers might need to have a background task running that process

6
gnocchi/storage/_carbonara.py
View File

@ -162,7 +162,7 @@ class CarbonaraBasedStorage(storage.StorageDriver):
(metric_id, self._pending_measures_to_process_count(metric_id))
for metric_id in metrics_to_process)
def process_measures(self, indexer):
def process_measures(self, indexer, sync=False):
metrics_to_process = self._list_metric_with_measures_to_process()
metrics = indexer.get_metrics(metrics_to_process)
# This build the list of deleted metrics, i.e. the metrics we have
@ -177,7 +177,7 @@ class CarbonaraBasedStorage(storage.StorageDriver):
# Do not block if we cannot acquire the lock, that means some other
# worker is doing the job. We'll just ignore this metric and may
# get back later to it if needed.
if lock.acquire(blocking=False):
if lock.acquire(blocking=sync):
try:
LOG.debug("Processing measures for %s" % metric)
with self._process_measure_for_metric(metric) as measures:
@ -234,6 +234,8 @@ class CarbonaraBasedStorage(storage.StorageDriver):
self._store_unaggregated_timeserie(metric,
ts.serialize())
except Exception:
if sync:
raise
LOG.error("Error processing new measures", exc_info=True)
finally:
lock.release()

2
gnocchi/tests/test_aggregates.py
View File

@ -60,7 +60,7 @@ class TestAggregates(tests_base.TestCase):
with mock.patch.object(self.index, 'get_metrics') as f:
f.return_value = [metric]
self.storage.process_background_tasks(self.index)
self.storage.process_background_tasks(self.index, True)
return metric

2
gnocchi/tests/test_rest.py
View File

@ -149,7 +149,7 @@ class TestingApp(webtest.TestApp):
if self.auth:
req.headers['X-Auth-Token'] = self.token
response = super(TestingApp, self).do_request(req, *args, **kwargs)
self.storage.process_background_tasks(self.indexer)
self.storage.process_background_tasks(self.indexer, True)
return response

8
gnocchi/tests/test_statsd.py
View File

@ -67,7 +67,7 @@ class TestStatsd(tests_base.TestCase):
metric = r.get_metric(metric_key)
self.stats.storage.process_background_tasks(self.stats.indexer)
self.stats.storage.process_background_tasks(self.stats.indexer, True)
measures = self.stats.storage.get_measures(metric)
self.assertEqual([
@ -86,7 +86,7 @@ class TestStatsd(tests_base.TestCase):
("127.0.0.1", 12345))
self.stats.flush()
self.stats.storage.process_background_tasks(self.stats.indexer)
self.stats.storage.process_background_tasks(self.stats.indexer, True)
measures = self.stats.storage.get_measures(metric)
self.assertEqual([
@ -117,7 +117,7 @@ class TestStatsd(tests_base.TestCase):
with_metrics=True)
metric = r.get_metric(metric_key)
self.stats.storage.process_background_tasks(self.stats.indexer)
self.stats.storage.process_background_tasks(self.stats.indexer, True)
measures = self.stats.storage.get_measures(metric)
self.assertEqual([
@ -134,7 +134,7 @@ class TestStatsd(tests_base.TestCase):
("127.0.0.1", 12345))
self.stats.flush()
self.stats.storage.process_background_tasks(self.stats.indexer)
self.stats.storage.process_background_tasks(self.stats.indexer, True)
measures = self.stats.storage.get_measures(metric)
self.assertEqual([

16
gnocchi/tests/test_storage.py
View File

@ -49,7 +49,7 @@ class TestStorageDriver(tests_base.TestCase):
])
with mock.patch.object(self.index, 'get_metrics') as f:
f.return_value = [self.metric]
self.storage.process_background_tasks(self.index)
self.storage.process_background_tasks(self.index, True)
self.storage.add_measures(self.metric, [
storage.Measure(datetime.datetime(2014, 1, 1, 13, 0, 1), 1),
@ -60,7 +60,7 @@ class TestStorageDriver(tests_base.TestCase):
side_effect=ValueError("boom!")):
with mock.patch('gnocchi.carbonara.msgpack.loads',
side_effect=ValueError("boom!")):
self.storage.process_background_tasks(self.index)
self.storage.process_background_tasks(self.index, True)
expected_calls = [
mock.call.debug('Processing measures for %s' % self.metric.id),
@ -86,7 +86,7 @@ class TestStorageDriver(tests_base.TestCase):
])
with mock.patch.object(self.index, 'get_metrics') as f:
f.return_value = [self.metric]
self.storage.process_background_tasks(self.index)
self.storage.process_background_tasks(self.index, True)
self.storage.delete_metric(self.metric)
def test_delete_nonempty_metric_unprocessed(self):
@ -129,7 +129,7 @@ class TestStorageDriver(tests_base.TestCase):
for i in six.moves.range(0, 60) for j in six.moves.range(0, 60)])
with mock.patch.object(self.index, 'get_metrics') as f:
f.return_value = [m]
self.storage.process_background_tasks(self.index)
self.storage.process_background_tasks(self.index, True)
self.assertEqual(3661, len(self.storage.get_measures(m)))
@ -142,7 +142,7 @@ class TestStorageDriver(tests_base.TestCase):
])
with mock.patch.object(self.index, 'get_metrics') as f:
f.return_value = [self.metric]
self.storage.process_background_tasks(self.index)
self.storage.process_background_tasks(self.index, True)
self.assertEqual([
(utils.datetime_utc(2014, 1, 1), 86400.0, 39.75),
@ -277,7 +277,7 @@ class TestStorageDriver(tests_base.TestCase):
])
with mock.patch.object(self.index, 'get_metrics') as f:
f.return_value = [self.metric, metric2]
self.storage.process_background_tasks(self.index)
self.storage.process_background_tasks(self.index, True)
values = self.storage.get_cross_metric_measures([self.metric, metric2])
self.assertEqual([
@ -337,7 +337,7 @@ class TestStorageDriver(tests_base.TestCase):
])
with mock.patch.object(self.index, 'get_metrics') as f:
f.return_value = [self.metric, metric2]
self.storage.process_background_tasks(self.index)
self.storage.process_background_tasks(self.index, True)
values = self.storage.get_cross_metric_measures([self.metric, metric2])
self.assertEqual([
@ -367,7 +367,7 @@ class TestStorageDriver(tests_base.TestCase):
])
with mock.patch.object(self.index, 'get_metrics') as f:
f.return_value = [self.metric, metric2]
self.storage.process_background_tasks(self.index)
self.storage.process_background_tasks(self.index, True)
self.assertEqual(
{metric2: [],