Merge "Emit per-branch queue stats separately"

doc/source/monitoring.rst

@@ -237,13 +237,17 @@ These metrics are emitted by the Zuul :ref:`scheduler`:
          This hierarchy holds more specific metrics for each
          :term:`project queue` in the pipeline.
 
-         .. stat:: <queue name>
+         .. stat:: <queue>
 
             The name of the queue.  If the queue is automatically
             generated for a single project, the name of the project is
             used by default.  Embedded ``.`` characters will be
             translated to ``_``, and ``/`` to ``.``.
 
+            If the queue is configured as per-branch, the metrics
+            below are omitted and instead found under
+            :stat:`zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.branch`.
+
             .. stat:: current_changes
                :type: gauge
 
@@ -266,6 +270,20 @@ These metrics are emitted by the Zuul :ref:`scheduler`:
 
                The number of changes processed by the queue.
 
+            .. stat:: branch
+
+               If the queue is configured as per-branch, this
+               hierarchy will be present and will hold stats for each
+               branch seen.
+
+               .. stat:: <branch>
+
+                  The name of the branch.  Embedded ``.`` characters
+                  will be translated to ``_``, and ``/`` to ``.``.
+
+                  Underneath this key are per-branch values of the
+                  metrics above.
+
       .. stat:: project
 
          This hierarchy holds more specific metrics for each project

releasenotes/notes/per-branch-stats-0c9f262a70d5cc81.yaml

@@ -0,0 +1,15 @@
+---
+fixes:
+  - |
+    Monitoring stats for per-branch queues are now distinct from
+    shared-branch queues.  Shared branch queue stats are at:
+
+      :stat:`zuul.tenant.<tenant>.pipeline.<pipeline>.queue`.
+
+    Per-branch queue stats are now at:
+
+      :stat:`zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.branch`.
+
+    Prior to this change, per-branch queue stats for one branch queue
+    may have overwritten the stats from another queue resulting in
+    incomplete or incorrect data.

tests/unit/test_scheduler.py

@@ -6804,6 +6804,21 @@ class TestChangeQueues(ZuulTestCase):
         known upfront and the queues are pre-seeded.
         """
         self._test_dependent_queues_per_branch('org/project')
+        self.assertReportedStat(
+            'zuul.tenant.tenant-one.pipeline.gate.queue.'
+            'integrated.branch.master.current_changes',
+            value='1', kind='g')
+        self.assertReportedStat(
+            'zuul.tenant.tenant-one.pipeline.gate.queue.'
+            'integrated.branch.master.window',
+            value='20', kind='g')
+        self.assertReportedStat(
+            'zuul.tenant.tenant-one.pipeline.gate.queue.'
+            'integrated.branch.master.resident_time', kind='ms')
+        self.assertReportedStat(
+            'zuul.tenant.tenant-one.pipeline.gate.queue.'
+            'integrated.branch.master.total_changes', value='1',
+            kind='c')
 
     def test_dependent_queues_per_branch_no_config(self):
         """

zuul/manager/__init__.py

@@ -2293,7 +2293,23 @@ class PipelineManager(metaclass=ABCMeta):
                 # stats.gauges.zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.total_changes
                 # stats.gauges.zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.current_changes
                 # stats.gauges.zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.window
-                queuekey = '%s.queue.%s' % (key, queuename)
+                queuekey = f'{key}.queue.{queuename}'
+
+                # Handle per-branch queues
+                layout = self.pipeline.tenant.layout
+                queue_config = layout.queues.get(item.queue.name)
+                per_branch = queue_config and queue_config.per_branch
+                if per_branch and item.queue.project_branches:
+                    # Get the first project-branch of this queue,
+                    # which is a tuple of project, branch, and get
+                    # second item of that tuple, the branch name.  In
+                    # a per-branch queue, we expect the branch name to
+                    # be the same for every project.
+                    branch = item.queue.project_branches[0][1]
+                    if branch:
+                        branch = branch.replace('.', '_').replace('/', '.')
+                        queuekey = f'{queuekey}.branch.{branch}'
+
                 queue_changes = sum(len(i.changes) for i in item.queue.queue)
                 self.sched.statsd.gauge(queuekey + '.current_changes',
                                         queue_changes)