openstack
/
goal-tools
Code Issues Proposed changes

add tool to extract job settings from project-config 

We want a tool to show the settings that we can move from project-config
into a project repo, and those we need to keep.

Change-Id: I2eac58f2d272cdec899df35b610c5a9cc0b878ba
Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann 2018-06-26 14:54:46 -04:00
parent f184b27ba0
commit 20378606b8
8 changed files with 789 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
README.rst
View File

@ -1,3 +1,46 @@
===========================
Community-wide Goal Tools
===========================
To use the tools, run ``tox -e venv --notest`` to create a virtualenv
with all of the dependencies. The tools will then be installed into
``.tox/venv/bin`` and can be run directly from there or via ``tox -e
venv -- COMMAND_NAME``.
who-helped
==========
``who-helped`` is a tool for looking at the contributor statistics for
a set of patches.
python3-first
=============
``python3-first`` is the parent command for a set of tools for
implementing the `python3-first goal
<https://review.openstack.org/#/c/575933/>`_.
The ``jobs extract`` sub-command reads the Zuul configuration from the
``openstack/project-config`` repository and then for a given
repository and branch prints the set of job definitions that should be
copied into that branch of that project.
.. code-block:: console
$ git clone git://git.openstack.org/openstack-infra/project-config
$ git clone git://git.openstack.org/openstack/goal-tools
$ cd goal-tools
$ tox -e venv -- python3-first jobs extract --project-config ../project-config \
openstack-dev/devstack stable/queens
The ``jobs retain`` sub-command reads the same Zuul configuration data
and prints the settings that need to stay in
``openstack/project-config``.
.. code-block:: console
$ tox -e venv -- python3-first jobs retain --project-config ../project-config \
openstack-dev/devstack
Use the ``-v`` option to python3-first to see debug information on
stderr (allowing stdout to be redirected to a file safely).

0
goal_tools/python3_first/__init__.py Normal file
View File

286
goal_tools/python3_first/jobs.py Executable file
View File

@ -0,0 +1,286 @@
#!/usr/bin/env python3
# Show the project settings for the repository that should be moved
# into the tree at a given branch.
import logging
import os.path
import re
from goal_tools.python3_first import projectconfig_ruamellib
from cliff import command
LOG = logging.getLogger(__name__)
# Items to keep in project-config.
KEEP = [
'system-required',
'translation-jobs',
]
BRANCHES = [
'stable/ocata',
'stable/pike',
'stable/queens',
'stable/rocky',
'master',
]
def branches_for_job(job_params):
branch_patterns = job_params.get('branches', [])
if not isinstance(branch_patterns, list):
branch_patterns = [branch_patterns]
for pattern in branch_patterns:
for branch in BRANCHES:
# LOG.debug('comparing %r with %r', branch, pattern)
if re.search(pattern, branch):
yield branch
def filter_jobs_on_branch(project, branch):
LOG.debug('filtering on %s', branch)
for queue, value in list(project.items()):
if not isinstance(value, dict):
continue
if queue == 'templates':
continue
if 'jobs' not in value:
continue
LOG.debug('%s queue', queue)
keep = []
for job in value['jobs']:
if not isinstance(job, dict):
keep.append(job)
continue
job_name = list(job.keys())[0]
job_params = list(job.values())[0]
if 'branches' not in job_params:
keep.append(job)
continue
branches = list(branches_for_job(job_params))
if not branches:
# The job is not applied to any branches.
LOG.debug('matches no branches, ignoring')
continue
LOG.debug('%s applies to branches: %s',
job_name, ', '.join(branches))
if branch not in branches:
# The job is not applied to the current branch.
want = False
elif len(branches) > 1:
# The job is applied to multiple branches, so if our
# branch is in that set we should go ahead and take
# it.
want = branch in branches
else:
# The job is applied to only 1 branch. If that branch
# is the master branch, we need to leave the setting
# in the project-config file.
want = branch != 'master'
if want:
LOG.debug('%s keeping', job_name)
del job_params['branches']
if not job_params:
# no parameters left, just add the job name
keep.append(job_name)
else:
keep.append(job)
else:
LOG.debug('%s ignoring', job_name)
if keep:
value['jobs'] = keep
else:
del value['jobs']
if not value:
del project[queue]
def find_templates_to_extract(project):
templates = project.get('templates', [])
to_keep = [
t
for t in templates
if t not in KEEP
]
if to_keep:
project['templates'] = to_keep
elif 'templates' in project:
del project['templates']
class JobsExtract(command.Command):
"show the project settings to extract for a repository"
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.add_argument(
'--project-config-dir',
default='../project-config',
help='the location of the project-config repo',
)
parser.add_argument(
'repo',
help='the repository name',
)
parser.add_argument(
'branch',
help='filter the settings by branch',
)
return parser
def take_action(self, parsed_args):
yaml = projectconfig_ruamellib.YAML()
project_filename = os.path.join(
parsed_args.project_config_dir,
'zuul.d',
'projects.yaml',
)
LOG.debug('loading project settings from %s', project_filename)
with open(project_filename, 'r', encoding='utf-8') as f:
project_settings = yaml.load(f)
LOG.debug('looking for settings for %s', parsed_args.repo)
for entry in project_settings:
if 'project' not in entry:
continue
if entry['project'].get('name') == parsed_args.repo:
break
else:
raise ValueError('Could not find {} in {}'.format(
parsed_args.repo, project_filename))
# Remove the items that need to stay in project-config.
find_templates_to_extract(entry['project'])
# Filter the jobs by branch.
if parsed_args.branch:
filter_jobs_on_branch(entry['project'], parsed_args.branch)
# Remove the 'name' value in case we can copy the results
# directly into a new file.
if 'name' in entry['project']:
del entry['project']['name']
print()
yaml.dump([entry], self.app.stdout)
def find_jobs_to_retain(project):
for queue, value in list(project.items()):
if not isinstance(value, dict):
continue
if queue == 'templates':
continue
if 'jobs' not in value:
continue
LOG.debug('%s queue', queue)
keep = []
for job in value['jobs']:
if not isinstance(job, dict):
continue
job_name = list(job.keys())[0]
job_params = list(job.values())[0]
if 'branches' not in job_params:
continue
branches = list(branches_for_job(job_params))
if not branches:
# The job is not applied to any branches.
LOG.debug('matches no branches, ignoring')
continue
LOG.debug('%s applies to branches: %s',
job_name, ', '.join(branches))
# If the job only applies to the master branch we need to
# keep it.
if branches == ['master']:
LOG.debug('%s keeping', job_name)
keep.append(job)
else:
LOG.debug('%s ignoring', job_name)
if keep:
value['jobs'] = keep
else:
del value['jobs']
if not value:
del project[queue]
def find_templates_to_retain(project):
# Remove the items that need to move to the project repo.
templates = project.get('templates', [])
to_keep = [
t
for t in templates
if t in KEEP
]
if to_keep:
project['templates'] = to_keep
elif 'templates' in project:
del project['templates']
class JobsRetain(command.Command):
"show the project settings to keep in project-config"
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.add_argument(
'--project-config-dir',
default='../project-config',
help='the location of the project-config repo',
)
parser.add_argument(
'repo',
help='the repository name',
)
return parser
def take_action(self, parsed_args):
yaml = projectconfig_ruamellib.YAML()
project_filename = os.path.join(
parsed_args.project_config_dir,
'zuul.d',
'projects.yaml',
)
LOG.debug('loading project settings from %s', project_filename)
with open(project_filename, 'r', encoding='utf-8') as f:
project_settings = yaml.load(f)
LOG.debug('looking for settings for %s', parsed_args.repo)
for entry in project_settings:
if 'project' not in entry:
continue
if entry['project'].get('name') == parsed_args.repo:
break
else:
raise ValueError('Could not find {} in {}'.format(
parsed_args.repo, project_filename))
find_templates_to_retain(entry['project'])
find_jobs_to_retain(entry['project'])
print()
yaml.dump([entry], self.app.stdout)

42
goal_tools/python3_first/main.py Normal file
View File

@ -0,0 +1,42 @@
#!/usr/bin/env python3
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import sys
from cliff import app
from cliff import commandmanager
import pbr.version
class Python3First(app.App):
"""Tool for working on the python3-first goal.
"""
def __init__(self):
version_info = pbr.version.VersionInfo('goal-tools')
super().__init__(
version=version_info.version_string(),
description='tool for working on python3-first goal',
command_manager=commandmanager.CommandManager('python3_first'),
deferred_help=False,
)
def initialize_app(self, argv):
# Quiet the urllib3 module output coming out of requests.
logging.getLogger('urllib3').setLevel(logging.WARNING)
def main(argv=sys.argv[1:]):
return Python3First().run(argv)

44
goal_tools/python3_first/projectconfig_ruamellib.py Executable file
View File

@ -0,0 +1,44 @@
# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import ruamel.yaml
def none_representer(dumper, data):
return dumper.represent_scalar('tag:yaml.org,2002:null', 'null')
class YAML(object):
def __init__(self):
self.yaml = ruamel.yaml.YAML()
self.yaml.allow_duplicate_keys = True
self.yaml.representer.add_representer(type(None), none_representer)
self.yaml.indent(mapping=2, sequence=4, offset=2)
def load(self, stream):
return self.yaml.load(stream)
def tr(self, x):
x = x.replace('\n-', '\n\n-')
newlines = []
for line in x.split('\n'):
if '#' in line:
newlines.append(line)
else:
newlines.append(line[2:])
return '\n'.join(newlines)
def dump(self, data, *args, **kwargs):
self.yaml.dump(data, *args, transform=self.tr, **kwargs)

365
goal_tools/tests/test_jobs.py Normal file
View File

@ -0,0 +1,365 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from goal_tools.python3_first import jobs
from goal_tools.tests import base
class TestBranchesForJob(base.TestCase):
def test_none(self):
params = {}
self.assertEqual(
[],
list(jobs.branches_for_job(params)),
)
def test_regex_string(self):
params = {
'branches': '^master',
}
self.assertEqual(
['master'],
list(jobs.branches_for_job(params)),
)
def test_regex_list(self):
params = {
'branches': [
'^master',
'ocata',
]
}
self.assertEqual(
['master', 'stable/ocata'],
list(jobs.branches_for_job(params)),
)
class TestFilterJobsOnBranch(base.TestCase):
def test_no_jobs(self):
project = {
'templates': [
'system-required',
'integrated-gate',
'integrated-gate-py35',
'publish-openstack-sphinx-docs',
],
}
expected = {
'templates': [
'system-required',
'integrated-gate',
'integrated-gate-py35',
'publish-openstack-sphinx-docs',
],
}
jobs.filter_jobs_on_branch(project, 'master')
self.assertEqual(expected, project)
def test_no_match(self):
project = {
'check': {
'jobs': [
{
'legacy-devstack-dsvm-updown': {
'branches': 'stable',
}
},
],
},
}
expected = {
}
jobs.filter_jobs_on_branch(project, 'master')
self.assertEqual(expected, project)
def test_no_regex(self):
project = {
'check': {
'jobs': [
'openstack-tox-bashate',
],
},
}
expected = {
'check': {
'jobs': [
'openstack-tox-bashate',
],
},
}
jobs.filter_jobs_on_branch(project, 'master')
self.assertEqual(expected['check']['jobs'], project['check']['jobs'])
def test_stable_match(self):
project = {
'check': {
'jobs': [
{
'legacy-devstack-dsvm-updown': {
'branches': '^stable',
}
},
],
},
}
expected = {
'check': {
'jobs': [
'legacy-devstack-dsvm-updown',
],
},
}
jobs.filter_jobs_on_branch(project, 'stable/ocata')
self.assertEqual(expected['check']['jobs'], project['check']['jobs'])
def test_master(self):
# Because legacy-devstack-dsvm-updown *only* matches the
# master branch the settings to include it need to stay in
# project-config.
project = {
'check': {
'jobs': [
'openstack-tox-bashate',
{
'legacy-devstack-dsvm-updown': {
'branches': '^(?!stable)',
}
},
],
},
}
expected = {
'check': {
'jobs': [
'openstack-tox-bashate',
],
},
}
jobs.filter_jobs_on_branch(project, 'master')
self.assertEqual(expected['check']['jobs'], project['check']['jobs'])
class TestFindJobsToRetain(base.TestCase):
def test_no_jobs(self):
project = {
'templates': [
'system-required',
'integrated-gate',
'integrated-gate-py35',
'publish-openstack-sphinx-docs',
],
'check': {
'jobs': [],
},
}
expected = {
'templates': [
'system-required',
'integrated-gate',
'integrated-gate-py35',
'publish-openstack-sphinx-docs',
],
}
jobs.find_jobs_to_retain(project)
self.assertEqual(expected, project)
def test_no_match(self):
project = {
'check': {
'jobs': [
'openstack-tox-bashate',
{
'legacy-devstack-dsvm-updown': {
'branches': '^stable',
}
},
],
},
}
expected = {
}
jobs.find_jobs_to_retain(project)
self.assertEqual(expected, project)
def test_stable_match(self):
project = {
'check': {
'jobs': [
'openstack-tox-bashate',
{
'legacy-devstack-dsvm-updown': {
'branches': '^stable',
}
},
],
},
}
expected = {
}
jobs.find_jobs_to_retain(project)
self.assertEqual(expected, project)
def test_master(self):
# Because legacy-devstack-dsvm-updown *only* matches the
# master branch the settings to include it need to stay in
# project-config.
project = {
'check': {
'jobs': [
'openstack-tox-bashate',
{
'legacy-devstack-dsvm-updown': {
'branches': '^(?!stable)',
}
},
],
},
}
expected = {
'check': {
'jobs': [
{
'legacy-devstack-dsvm-updown': {
'branches': '^(?!stable)',
}
},
],
},
}
jobs.find_jobs_to_retain(project)
self.assertEqual(expected['check']['jobs'], project['check']['jobs'])
class TestJobsExtractTemplates(base.TestCase):
def test_no_templates(self):
project = {
}
expected = {
}
jobs.find_templates_to_extract(project)
self.assertEqual(expected, project)
def test_no_templates_remain(self):
project = {
'templates': [
'system-required',
'translation-jobs',
],
}
expected = {
}
jobs.find_templates_to_extract(project)
self.assertEqual(expected, project)
def test_system_required(self):
project = {
'templates': [
'system-required',
'integrated-gate',
'integrated-gate-py35',
'publish-openstack-sphinx-docs',
],
}
expected = {
'templates': [
'integrated-gate',
'integrated-gate-py35',
'publish-openstack-sphinx-docs',
],
}
jobs.find_templates_to_extract(project)
self.assertEqual(expected, project)
def test_translation_jobs(self):
project = {
'templates': [
'translation-jobs',
'integrated-gate',
'integrated-gate-py35',
'publish-openstack-sphinx-docs',
],
}
expected = {
'templates': [
'integrated-gate',
'integrated-gate-py35',
'publish-openstack-sphinx-docs',
],
}
jobs.find_templates_to_extract(project)
self.assertEqual(expected, project)
class TestJobsRetainTemplates(base.TestCase):
def test_no_templates(self):
project = {
}
expected = {
}
jobs.find_templates_to_retain(project)
self.assertEqual(expected, project)
def test_no_templates_remain(self):
project = {
'templates': [
'system-required',
'translation-jobs',
],
}
expected = {
'templates': [
'system-required',
'translation-jobs',
],
}
jobs.find_templates_to_retain(project)
self.assertEqual(expected, project)
def test_system_required(self):
project = {
'templates': [
'system-required',
'integrated-gate',
'integrated-gate-py35',
'publish-openstack-sphinx-docs',
],
}
expected = {
'templates': [
'system-required',
],
}
jobs.find_templates_to_retain(project)
self.assertEqual(expected, project)
def test_translation_jobs(self):
project = {
'templates': [
'translation-jobs',
'integrated-gate',
'integrated-gate-py35',
'publish-openstack-sphinx-docs',
],
}
expected = {
'templates': [
'translation-jobs',
],
}
jobs.find_templates_to_retain(project)
self.assertEqual(expected, project)

9
requirements.txt
View File

@ -1,8 +1,9 @@
# the storyboardclient lib is not on PyPI yet
# python-storyboardclient
appdirs>=1.4.3
beautifulsoup4>=4.6.0
requests
pyyaml
cliff
python-storyboardclient
pyyaml
requests
ruamel.yaml
six
yamlordereddictloader

4
setup.cfg
View File

@ -27,6 +27,7 @@ setup-hooks =
console_scripts =
import-goal = goal_tools.import_goal:main
who-helped = goal_tools.who_helped.main:main
python3-first = goal_tools.python3_first.main:main
who_helped =
contributions list = goal_tools.who_helped.contributions:ListContributions
@ -42,6 +43,9 @@ who_helped =
top list = goal_tools.who_helped.top:TopN
team show = goal_tools.who_helped.team:ShowTeam
python3_first =
jobs extract = goal_tools.python3_first.jobs:JobsExtract
jobs retain = goal_tools.python3_first.jobs:JobsRetain
[wheel]
universal = 1