diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py index ba19314362..5198972ef2 100644 --- a/tests/unit/test_web.py +++ b/tests/unit/test_web.py @@ -1377,6 +1377,69 @@ class TestWebMultiTenant(BaseTestWeb): sorted(["tenant-one", "tenant-two", "tenant-four"])) +class TestWebGlobalSemaphores(BaseTestWeb): + tenant_config_file = 'config/global-semaphores-config/main.yaml' + + def test_web_semaphores(self): + self.executor_server.hold_jobs_in_build = True + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertBuilds([ + dict(name='test-global-semaphore', changes='1,1'), + dict(name='test-common-semaphore', changes='1,1'), + dict(name='test-project1-semaphore', changes='1,1'), + dict(name='test-global-semaphore', changes='2,1'), + dict(name='test-common-semaphore', changes='2,1'), + dict(name='test-project2-semaphore', changes='2,1'), + ]) + + tenant1_buildset_uuid = self.builds[0].parameters['zuul']['buildset'] + data = self.get_url('api/tenant/tenant-one/semaphores').json() + + expected = [ + {'name': 'common-semaphore', + 'global': False, + 'max': 10, + 'holders': { + 'count': 1, + 'this_tenant': [ + {'buildset_uuid': tenant1_buildset_uuid, + 'job_name': 'test-common-semaphore'} + ], + 'other_tenants': 0 + }}, + {'name': 'global-semaphore', + 'global': True, + 'max': 100, + 'holders': { + 'count': 2, + 'this_tenant': [ + {'buildset_uuid': tenant1_buildset_uuid, + 'job_name': 'test-global-semaphore'} + ], + 'other_tenants': 1 + }}, + {'name': 'project1-semaphore', + 'global': False, + 'max': 11, + 'holders': { + 'count': 1, + 'this_tenant': [ + {'buildset_uuid': tenant1_buildset_uuid, + 'job_name': 'test-project1-semaphore'} + ], + 'other_tenants': 0 + }} + ] + self.assertEqual(expected, data) + + class TestEmptyConfig(BaseTestWeb): tenant_config_file = 'config/empty-config/main.yaml' diff --git a/zuul/configloader.py b/zuul/configloader.py index 365967d563..a6ad799d90 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -1722,10 +1722,11 @@ class TenantParser(object): tenant.layout = self._parseLayout( tenant, parsed_config, loading_errors, layout_uuid) + tenant.semaphore_handler = SemaphoreHandler( + self.zk_client, self.statsd, tenant.name, tenant.layout, abide, + read_only=(not bool(self.scheduler)) + ) if self.scheduler: - tenant.semaphore_handler = SemaphoreHandler( - self.zk_client, self.statsd, tenant.name, tenant.layout, abide - ) # Only call the postConfig hook if we have a scheduler as this will # change data in ZooKeeper. In case we are in a zuul-web context, # we don't want to do that. diff --git a/zuul/model.py b/zuul/model.py index 0d889f5576..25d657bb6b 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -659,6 +659,13 @@ class PipelineState(zkobject.ZKObject): safe_pipeline = urllib.parse.quote_plus(pipeline.name) return f"/zuul/tenant/{safe_tenant}/pipeline/{safe_pipeline}" + @classmethod + def parsePath(self, path): + """Return path components for use by the REST API""" + root, safe_tenant, pipeline, safe_pipeline = path.rsplit('/', 3) + return (urllib.parse.unquote_plus(safe_tenant), + urllib.parse.unquote_plus(safe_pipeline)) + def _dirtyPath(self): return f'{self.getPath()}/dirty' @@ -3871,6 +3878,13 @@ class BuildSet(zkobject.ZKObject): def getPath(self): return f"{self.item.getPath()}/buildset/{self.uuid}" + @classmethod + def parsePath(self, path): + """Return path components for use by the REST API""" + item_path, bs, uuid = path.rsplit('/', 2) + tenant, pipeline, item_uuid = QueueItem.parsePath(item_path) + return (tenant, pipeline, item_uuid, uuid) + def serialize(self, context): data = { # "item": self.item, @@ -4285,6 +4299,13 @@ class QueueItem(zkobject.ZKObject): def itemPath(cls, pipeline_path, item_uuid): return f"{pipeline_path}/item/{item_uuid}" + @classmethod + def parsePath(self, path): + """Return path components for use by the REST API""" + pipeline_path, item, uuid = path.rsplit('/', 2) + tenant, pipeline = PipelineState.parsePath(pipeline_path) + return (tenant, pipeline, uuid) + def serialize(self, context): if isinstance(self.event, TriggerEvent): event_type = "TriggerEvent" diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py index 644b82becc..c2b6a4ddfc 100755 --- a/zuul/web/__init__.py +++ b/zuul/web/__init__.py @@ -47,6 +47,7 @@ from zuul.lib.monitoring import MonitoringServer from zuul.lib.re2util import filter_allowed_disallowed from zuul.model import ( Abide, + BuildSet, Branch, ChangeQueue, DequeueEvent, @@ -796,6 +797,7 @@ class ZuulWebAPI(object): 'project/{project:.*}/branch/{branch:.*}/' 'freeze-jobs', 'pipelines': '/api/tenant/{tenant}/pipelines', + 'semaphores': '/api/tenant/{tenant}/semaphores', 'labels': '/api/tenant/{tenant}/labels', 'nodes': '/api/tenant/{tenant}/nodes', 'key': '/api/tenant/{tenant}/key/{project:.*}.pub', @@ -1533,6 +1535,44 @@ class ZuulWebAPI(object): resp.headers['Access-Control-Allow-Origin'] = '*' return data + @cherrypy.expose + @cherrypy.tools.save_params() + @cherrypy.tools.json_out( + content_type='application/json; charset=utf-8', handler=json_handler, + ) + def semaphores(self, tenant_name): + tenant = self._getTenantOrRaise(tenant_name) + result = [] + names = set(tenant.layout.semaphores.keys()) + names = names.union(tenant.global_semaphores) + for semaphore_name in sorted(names): + semaphore = tenant.layout.getSemaphore( + self.zuulweb.abide, semaphore_name) + holders = tenant.semaphore_handler.semaphoreHolders(semaphore_name) + this_tenant = [] + other_tenants = 0 + for holder in holders: + (holder_tenant, holder_pipeline, + holder_item_uuid, holder_buildset_uuid + ) = BuildSet.parsePath(holder['buildset_path']) + if holder_tenant != tenant_name: + other_tenants += 1 + continue + this_tenant.append({'buildset_uuid': holder_buildset_uuid, + 'job_name': holder['job_name']}) + sem_out = {'name': semaphore.name, + 'global': semaphore.global_scope, + 'max': semaphore.max, + 'holders': { + 'count': len(this_tenant) + other_tenants, + 'this_tenant': this_tenant, + 'other_tenants': other_tenants}, + } + result.append(sem_out) + resp = cherrypy.response + resp.headers['Access-Control-Allow-Origin'] = '*' + return result + @cherrypy.expose @cherrypy.tools.save_params() @cherrypy.tools.websocket(handler_cls=LogStreamHandler) @@ -1864,6 +1904,8 @@ class ZuulWeb(object): controller=api, action='status') route_map.connect('api', '/api/tenant/{tenant}/status/change/{change}', controller=api, action='status_change') + route_map.connect('api', '/api/tenant/{tenant_name}/semaphores', + controller=api, action='semaphores') route_map.connect('api', '/api/tenant/{tenant_name}/jobs', controller=api, action='jobs') route_map.connect('api', '/api/tenant/{tenant_name}/job/{job_name}', diff --git a/zuul/zk/semaphore.py b/zuul/zk/semaphore.py index 721a0438a7..b45612c04f 100644 --- a/zuul/zk/semaphore.py +++ b/zuul/zk/semaphore.py @@ -41,8 +41,12 @@ class SemaphoreHandler(ZooKeeperSimpleBase): semaphore_root = "/zuul/semaphores" global_semaphore_root = "/zuul/global-semaphores" - def __init__(self, client, statsd, tenant_name, layout, abide): + def __init__(self, client, statsd, tenant_name, layout, abide, + read_only=False): super().__init__(client) + if read_only: + statsd = None + self.read_only = read_only self.abide = abide self.layout = layout self.statsd = statsd @@ -71,6 +75,8 @@ class SemaphoreHandler(ZooKeeperSimpleBase): self.log.exception("Unable to send semaphore stats:") def acquire(self, item, job, request_resources): + if self.read_only: + raise RuntimeError("Read-only semaphore handler") if not job.semaphores: return True @@ -190,6 +196,8 @@ class SemaphoreHandler(ZooKeeperSimpleBase): break def release(self, sched, item, job, quiet=False): + if self.read_only: + raise RuntimeError("Read-only semaphore handler") if not job.semaphores: return @@ -242,6 +250,8 @@ class SemaphoreHandler(ZooKeeperSimpleBase): return 1 if semaphore is None else semaphore.max def cleanupLeaks(self): + if self.read_only: + raise RuntimeError("Read-only semaphore handler") # MODEL_API: >1 if COMPONENT_REGISTRY.model_api < 2: self.log.warning("Skipping semaphore cleanup since minimum model "