From 7ab338708059f576d9e34c3a9c08acd249165f46 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 5 Jul 2017 15:59:41 -0400 Subject: [PATCH] validate service types used for documentation jobs We want the service type use to be consistent across all of the various repos where we refer to it. This patch adds validation to ensure that service types associated with jobs for repositories match the service types assigned by the registry. In the process, it also changes the job definitions for some of the projects that were publishing to the "wrong" location. A separate patch will establish redirects from the old locations to the new locations. Some of the projects were not listed in the registry at all, so this patch depends on other patches that add the missing entries. The validation class defined here will eventually move to the os-service-types library, when that repository is ready to receive code. Change-Id: I2785ca8782b592c7af574e0b5a41407610873349 Depends-On: I27765d7760352f4c589168037ddd2e6792d1561f Depends-On: Ie530edd8aa40e5b89f997526e68117cafa68ee9c Signed-off-by: Doug Hellmann --- jenkins/jobs/projects.yaml | 22 +++--- tools/jenkins-projects-checks.py | 118 +++++++++++++++++++++++++++++++ tox.ini | 1 + 3 files changed, 130 insertions(+), 11 deletions(-) diff --git a/jenkins/jobs/projects.yaml b/jenkins/jobs/projects.yaml index 2af7c4a707..df349961ea 100644 --- a/jenkins/jobs/projects.yaml +++ b/jenkins/jobs/projects.yaml @@ -225,7 +225,7 @@ - 'gate-telemetry-dsvm-integration-{name}-{node}': node: ubuntu-xenial - install-guide-jobs: - service: telemetry-alarming + service: alarm - project: @@ -705,7 +705,7 @@ node: ubuntu-xenial - periodic-python-jobs-with-oslo-master - install-guide-jobs: - service: telemetry + service: meter - project: name: ceilometer-powervm @@ -6607,9 +6607,9 @@ envlist: genconfig node: ubuntu-xenial - api-ref-jobs: - service: shared-file-systems + service: shared-file-system - install-guide-jobs: - service: shared-file-systems + service: shared-file-system - '{pipeline}-manilaclient-dsvm-neutron-functional-{node}{suffix}': node: ubuntu-xenial pipeline: 'gate' @@ -8259,9 +8259,9 @@ node: ubuntu-xenial suffix: '' branch-override: default - # networking api-ref is maintained in neutron-lib repo. + # network api-ref is maintained in neutron-lib repo. - api-ref-jobs: - service: networking + service: network - project: name: neutron-specs @@ -13426,7 +13426,7 @@ - openstack-publish-jobs - openstack-releasenotes-jobs - api-ref-jobs: - service: clustering + service: resource-cluster - project: name: senlin-dashboard @@ -13915,9 +13915,9 @@ - translation-jobs-newton - translation-jobs-ocata - api-ref-jobs: - service: object-storage + service: object-store - install-guide-jobs: - service: object-storage + service: object-store - 'gate-{name}-tox-xfs-tmp-{envlist}-{node}': envlist: - py27 @@ -15440,9 +15440,9 @@ branch-override: default backend: redis - api-ref-jobs: - service: messaging + service: message - install-guide-jobs: - service: messaging + service: message - project: name: zaqar-specs diff --git a/tools/jenkins-projects-checks.py b/tools/jenkins-projects-checks.py index 4a30a5b264..c98dccd22a 100755 --- a/tools/jenkins-projects-checks.py +++ b/tools/jenkins-projects-checks.py @@ -14,9 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import io import glob import sys +import requests import voluptuous as v # The files uses YAML extensions like !include, therefore use the @@ -200,8 +202,124 @@ def _check_tox_builder(schema, entry): return count +class ServiceTypeValidator(object): + # NOTE(dhellmann): This class will move to os-service-types when + # that repo is ready to start accepting code. + + # The location of the service-types-authority data. + _URL = 'https://service-types.openstack.org/service-types.json' # noqa + + def __init__(self): + # FIXME(dhellmann): Improve error handling and add caching. + self._raw = requests.get(self._URL).json() + # Store a mapping from the project name to the service + # data. Include api_reference_project names when present so + # that validation code can look up either type of project. + by_project = {} + for s in self._raw['services']: + for key in ['project', 'api_reference_project']: + name = s.get(key) + if name: + by_project[self._canonical_project_name(name)] = s + self._raw['by_project'] = by_project + + def _canonical_project_name(self, name): + "Convert repo name to project name." + return name.rpartition('/')[-1] + + @property + def url(self): + "The URL from which the data was retrieved." + return self._URL + + @property + def version(self): + "The version of the data." + return self._raw['version'] + + @property + def forward(self): + "Mapping service type names to their aliases." + return copy.deepcopy(self._raw['forward']) + + @property + def reverse(self): + "Mapping aliases to their service type names." + return copy.deepcopy(self._raw['reverse']) + + @property + def services(self): + return copy.deepcopy(self._raw['services']) + + def get_data_for_project(self, name): + """Return the data value associated with the project. + + :param name: A repository or project name in the form + ``'openstack/project'`` or just ``'project'``. + :type name: str + :returns: dict + :raises: ValueError + + """ + key = name.rpartition('/')[-1] + try: + return self._raw['by_project'][key] + except KeyError: + raise ValueError( + 'No service_type was found for {}'.format(key), + ) + + +# The jobs for which the service type needs to be checked +_API_JOBS = ['install-guide-jobs', 'api-guide-jobs', 'api-ref-jobs'] + + +def validate_service_types(): + print("Validating Service Types") + print("========================") + count = 0 + # Load the current service-type-authority data + service_types = ServiceTypeValidator() + # Load the project job definitions + with io.open('jenkins/jobs/projects.yaml', 'r', encoding='utf-8') as f: + file_contents = local_yaml.load(f.read()) + for item in file_contents: + project = item.get('project', {}) + for job in project.get('jobs', []): + for api_job in _API_JOBS: + if api_job not in job: + continue + try: + proj_data = service_types.get_data_for_project( + project['name']) + except ValueError: + print('ERROR: Found service type reference "{}" for {} in {} ' + 'but not in authority list {}'.format( + job[api_job]['service'], + api_job, + project['name'], + service_types.url)) + count +=1 + else: + actual = job[api_job]['service'] + expected = proj_data['service_type'] + if actual != expected: + print('ERROR: Found service "{}" for {} ' + 'in {} but expected "{}"'.format( + job[api_job]['service'], + api_job, + project['name'], + expected)) + count += 1 + print('Found {} errors in service type settings ' + 'in jenkins/jobs/projects.yaml\n'.format( + count)) + return count + + def check_all(): errors = validate_jobs() + errors = errors or validate_service_types() # skip if formatting errors errors = check_alphabetical() or errors if errors: diff --git a/tox.ini b/tox.ini index 7db3dc1b09..1b201866d9 100644 --- a/tox.ini +++ b/tox.ini @@ -70,6 +70,7 @@ deps = PyYAML voluptuous jenkins-job-builder + requests commands = {toxinidir}/tools/jenkins-projects-checks.py