From 794545fc6439c8d9225a605cbc56c8331c21f6d0 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 28 Feb 2024 13:51:04 -0800 Subject: [PATCH] Emit per-branch queue stats separately We currently emit 4 statsd metrics for each shared queue, but in the case that a queue is configured as per-branch, we disregard the branch and emit the stats under the same hierarchy for any branch of that queue. This means that if we have a queue for integrated-master and a queue for integrated-stable at the same time, we would emit the stats for the master queue, then immediately emit the same stats for the stable queue, overwriting the master stats. To correct this, move the metrics down a level in the case that the queue is configured per-branch, and include the branch name in the key. Change-Id: I2f4b22394bc3774410a02ae76281eddf080e5c7f --- doc/source/monitoring.rst | 20 ++++++++++++++++++- .../per-branch-stats-0c9f262a70d5cc81.yaml | 15 ++++++++++++++ tests/unit/test_scheduler.py | 15 ++++++++++++++ zuul/manager/__init__.py | 18 ++++++++++++++++- 4 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/per-branch-stats-0c9f262a70d5cc81.yaml diff --git a/doc/source/monitoring.rst b/doc/source/monitoring.rst index 5bed8819f8..74796137eb 100644 --- a/doc/source/monitoring.rst +++ b/doc/source/monitoring.rst @@ -237,13 +237,17 @@ These metrics are emitted by the Zuul :ref:`scheduler`: This hierarchy holds more specific metrics for each :term:`project queue` in the pipeline. - .. stat:: + .. stat:: The name of the queue. If the queue is automatically generated for a single project, the name of the project is used by default. Embedded ``.`` characters will be translated to ``_``, and ``/`` to ``.``. + If the queue is configured as per-branch, the metrics + below are omitted and instead found under + :stat:`zuul.tenant..pipeline..queue..branch`. + .. stat:: current_changes :type: gauge @@ -266,6 +270,20 @@ These metrics are emitted by the Zuul :ref:`scheduler`: The number of changes processed by the queue. + .. stat:: branch + + If the queue is configured as per-branch, this + hierarchy will be present and will hold stats for each + branch seen. + + .. stat:: + + The name of the branch. Embedded ``.`` characters + will be translated to ``_``, and ``/`` to ``.``. + + Underneath this key are per-branch values of the + metrics above. + .. stat:: project This hierarchy holds more specific metrics for each project diff --git a/releasenotes/notes/per-branch-stats-0c9f262a70d5cc81.yaml b/releasenotes/notes/per-branch-stats-0c9f262a70d5cc81.yaml new file mode 100644 index 0000000000..5850609968 --- /dev/null +++ b/releasenotes/notes/per-branch-stats-0c9f262a70d5cc81.yaml @@ -0,0 +1,15 @@ +--- +fixes: + - | + Monitoring stats for per-branch queues are now distinct from + shared-branch queues. Shared branch queue stats are at: + + :stat:`zuul.tenant..pipeline..queue`. + + Per-branch queue stats are now at: + + :stat:`zuul.tenant..pipeline..queue..branch`. + + Prior to this change, per-branch queue stats for one branch queue + may have overwritten the stats from another queue resulting in + incomplete or incorrect data. diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index 3a33e781d1..0d4d8dcbac 100644 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -6804,6 +6804,21 @@ class TestChangeQueues(ZuulTestCase): known upfront and the queues are pre-seeded. """ self._test_dependent_queues_per_branch('org/project') + self.assertReportedStat( + 'zuul.tenant.tenant-one.pipeline.gate.queue.' + 'integrated.branch.master.current_changes', + value='1', kind='g') + self.assertReportedStat( + 'zuul.tenant.tenant-one.pipeline.gate.queue.' + 'integrated.branch.master.window', + value='20', kind='g') + self.assertReportedStat( + 'zuul.tenant.tenant-one.pipeline.gate.queue.' + 'integrated.branch.master.resident_time', kind='ms') + self.assertReportedStat( + 'zuul.tenant.tenant-one.pipeline.gate.queue.' + 'integrated.branch.master.total_changes', value='1', + kind='c') def test_dependent_queues_per_branch_no_config(self): """ diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py index c21cd1d455..871e6c1c3f 100644 --- a/zuul/manager/__init__.py +++ b/zuul/manager/__init__.py @@ -2269,7 +2269,23 @@ class PipelineManager(metaclass=ABCMeta): # stats.gauges.zuul.tenant..pipeline..queue..total_changes # stats.gauges.zuul.tenant..pipeline..queue..current_changes # stats.gauges.zuul.tenant..pipeline..queue..window - queuekey = '%s.queue.%s' % (key, queuename) + queuekey = f'{key}.queue.{queuename}' + + # Handle per-branch queues + layout = self.pipeline.tenant.layout + queue_config = layout.queues.get(item.queue.name) + per_branch = queue_config and queue_config.per_branch + if per_branch and item.queue.project_branches: + # Get the first project-branch of this queue, + # which is a tuple of project, branch, and get + # second item of that tuple, the branch name. In + # a per-branch queue, we expect the branch name to + # be the same for every project. + branch = item.queue.project_branches[0][1] + if branch: + branch = branch.replace('.', '_').replace('/', '.') + queuekey = f'{queuekey}.branch.{branch}' + queue_changes = sum(len(i.changes) for i in item.queue.queue) self.sched.statsd.gauge(queuekey + '.current_changes', queue_changes)