Ability to define batch size off polled samples.

Currently if batch_polled_samples=True (default) then it will batch
up all samples of a poller into one message. This can be problematic
if the poller returns a large number of samples.

Change-Id: Ifc60a0cb81876f6d9054cae82ba6579707a02f67
Closes-bug: #1770310
This commit is contained in:
Sam Morrison 2018-05-10 13:52:04 +10:00
parent b0c735dce1
commit 2dc21a5f05
3 changed files with 40 additions and 6 deletions

View File

@ -45,9 +45,13 @@ LOG = log.getLogger(__name__)
OPTS = [
cfg.BoolOpt('batch_polled_samples',
default=True,
deprecated_for_removal=True,
help='To reduce polling agent load, samples are sent to the '
'notification agent in a batch. To gain higher '
'throughput at the cost of load set this to False.'),
'throughput at the cost of load set this to False. '
'This option is deprecated, to disable batching set '
'batch_size = 0 in the polling group.'
),
]
POLLING_OPTS = [
@ -62,6 +66,10 @@ POLLING_OPTS = [
'config files. For each sub-group of the agent '
'pool with the same partitioning_group_prefix a disjoint '
'subset of pollsters should be loaded.'),
cfg.IntOpt('batch_size',
default=50,
help='Batch size of samples to send to notification agent, '
'Set to 0 to disable'),
]
@ -132,6 +140,12 @@ class PollingTask(object):
self.resources = collections.defaultdict(resource_factory)
self._batch = self.manager.conf.batch_polled_samples
self._batch_size = self.manager.conf.polling.batch_size
if not self._batch:
# Support deprecated way of disabling baching
self._batch_size = 0
self._telemetry_secret = self.manager.conf.publisher.telemetry_secret
def add(self, pollster, source):
@ -194,7 +208,10 @@ class PollingTask(object):
publisher_utils.meter_message_from_counter(
sample, self._telemetry_secret
))
if self._batch:
if self._batch_size:
if len(sample_batch) >= self._batch_size:
self._send_notification(sample_batch)
sample_batch = []
sample_batch.append(sample_dict)
else:
self._send_notification([sample_dict])

View File

@ -791,13 +791,17 @@ class TestPollingAgent(BaseAgent):
res_list="[<NovaLikeServer: unknown-name>]",
source=source_name))
def test_batching_polled_samples_false(self):
def test_batching_polled_samples_false_deprecated(self):
self.CONF.set_override('batch_polled_samples', False)
self._batching_samples(4, 4)
def test_batching_polled_samples_true(self):
self.CONF.set_override('batch_polled_samples', True)
self._batching_samples(4, 1)
def test_batching_polled_samples_disable_batch(self):
self.CONF.set_override('batch_size', 0, group='polling')
self._batching_samples(4, 4)
def test_batching_polled_samples_batch_size(self):
self.CONF.set_override('batch_size', 2, group='polling')
self._batching_samples(4, 2)
def test_batching_polled_samples_default(self):
self._batching_samples(4, 1)

View File

@ -0,0 +1,13 @@
---
features:
- >
Add support for configuring the size of samples the poller will send in
each batch.
upgrade:
- >
batch_size option added to [polling] section of configuration.
Use batch_size=0 to disable batching of samples.
deprecations:
- >
The option batch_polled_samples in the [DEFAULT] section is deprecated.
Use batch_size option in [polling] to configure and/or disable batching.