openstack
/
deb-gnocchi
Code Issues Proposed changes

ceph: async write new measures 

we should use async writes since we're not using threads anymore.

testing on small ceph cluster and posting 20metrics/POST results in
~2x better write performance.

Change-Id: Ic451e6ea874a2a3695d3d148b1c05ecc54aa473b
This commit is contained in:
gord chung 2017-05-15 17:12:05 +00:00
parent 5c18a8d4fe
commit 63fa106d78
1 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
gnocchi/storage/incoming/ceph.py
View File

@ -79,6 +79,7 @@ class CephStorage(_carbonara.CarbonaraBasedStorage):
def add_measures_batch(self, metrics_and_measures):
names_by_sack = defaultdict(list)
ops = []
for metric, measures in six.iteritems(metrics_and_measures):
name = "_".join((
self.MEASURE_PREFIX,
@ -88,8 +89,12 @@ class CephStorage(_carbonara.CarbonaraBasedStorage):
sack = self.get_sack_name(self.sack_for_metric(metric.id))
names_by_sack[sack].append(name)
data = self._encode_measures(measures)
self.ioctx.write_full(name, data)
ops.append(self.ioctx.aio_write_full(name, data))
while ops:
op = ops.pop()
op.wait_for_complete()
ops = []
for sack, names in names_by_sack.items():
with rados.WriteOpCtx() as op:
# NOTE(sileht): list all objects in a pool is too slow with
@ -99,8 +104,11 @@ class CephStorage(_carbonara.CarbonaraBasedStorage):
# omap the list of objects to process (not xattr because
# it doesn't # allow to configure the locking behavior)
self.ioctx.set_omap(op, tuple(names), (b"",) * len(names))
self.ioctx.operate_write_op(op, sack,
flags=self.OMAP_WRITE_FLAGS)
ops.append(self.ioctx.operate_aio_write_op(
op, sack, flags=self.OMAP_WRITE_FLAGS))
while ops:
op = ops.pop()
op.wait_for_complete()
def _build_report(self, details):
metrics = set()