diff --git a/goal_tools/gitutils.py b/goal_tools/gitutils.py new file mode 100644 index 0000000..e8b9f3d --- /dev/null +++ b/goal_tools/gitutils.py @@ -0,0 +1,31 @@ +import logging +import os.path +import subprocess + +LOG = logging.getLogger(__name__) + +start_dir = os.getcwd() +tools_dir = os.path.join(start_dir, 'tools') + + +def clone_repo(workdir, repo): + LOG.info('cloning %s', repo) + repo_dir = os.path.join(workdir, repo) + if os.path.exists(repo_dir): + raise RuntimeError('Found another copy of {} at {}'.format( + repo, repo_dir)) + subprocess.run( + [os.path.join(tools_dir, 'clone_repo.sh'), + '--workspace', workdir, + repo], + check=True, + ) + return os.path.join(workdir, repo) + + +def git(repo_dir, *args): + subprocess.run( + ['git'] + list(args), + check=True, + cwd=repo_dir, + ) diff --git a/goal_tools/python3_first/toxsettings.py b/goal_tools/python3_first/toxsettings.py index 0d41d12..7b97ac5 100644 --- a/goal_tools/python3_first/toxsettings.py +++ b/goal_tools/python3_first/toxsettings.py @@ -10,6 +10,7 @@ import subprocess from cliff import command from cliff import lister +from goal_tools import gitutils from goal_tools import governance LOG = logging.getLogger(__name__) @@ -130,32 +131,6 @@ class ToxMissingPy3(lister.Lister): return (columns, data) -start_dir = os.getcwd() -tools_dir = os.path.join(start_dir, 'tools') - - -def clone_repo(workdir, repo): - LOG.info('cloning %s', repo) - repo_dir = os.path.join(workdir, repo) - if os.path.exists(repo_dir): - raise RuntimeError('Found another copy of {} at {}'.format( - repo, repo_dir)) - subprocess.run( - [os.path.join(tools_dir, 'clone_repo.sh'), - '--workspace', workdir, - repo], - check=True, - ) - - -def git(repo_dir, *args): - subprocess.run( - ['git'] + list(args), - check=True, - cwd=repo_dir, - ) - - COMMIT_MESSAGE = '''\ fix tox python3 overrides @@ -176,8 +151,8 @@ Signed-off-by: Doug Hellmann def fix_one(workdir, repo, bad_envs): LOG.info('processing %s', repo) repo_dir = os.path.join(workdir, repo) - git(repo_dir, 'checkout', 'master') - git(repo_dir, 'checkout', '-b', 'python3-first-tox') + gitutils.git(repo_dir, 'checkout', 'master') + gitutils.git(repo_dir, 'checkout', '-b', 'python3-first-tox') tox_file = os.path.join(repo_dir, 'tox.ini') with open(tox_file, 'r', encoding='utf-8') as f: tox_contents = f.read() @@ -198,11 +173,11 @@ def fix_one(workdir, repo, bad_envs): ) with open(tox_file, 'w', encoding='utf-8') as f: f.write(tox_contents) - git(repo_dir, 'diff') - git(repo_dir, 'add', 'tox.ini') - git(repo_dir, 'review', '-s') - git(repo_dir, 'commit', '-m', COMMIT_MESSAGE) - git(repo_dir, 'show') + gitutils.git(repo_dir, 'diff') + gitutils.git(repo_dir, 'add', 'tox.ini') + gitutils.git(repo_dir, 'review', '-s') + gitutils.git(repo_dir, 'commit', '-m', COMMIT_MESSAGE) + gitutils.git(repo_dir, 'show') class ToxFixMissingPy3(command.Command): @@ -244,7 +219,7 @@ class ToxFixMissingPy3(command.Command): tracking_file = os.path.join(team_dir, 'master') - clone_repo(team_dir, r) + gitutils.clone_repo(team_dir, r) bad_envs = [ env diff --git a/goal_tools/python3_first/wheelsettings.py b/goal_tools/python3_first/wheelsettings.py index 5a4c9f7..b627a58 100644 --- a/goal_tools/python3_first/wheelsettings.py +++ b/goal_tools/python3_first/wheelsettings.py @@ -3,9 +3,12 @@ import configparser import logging import os.path +import shutil +from cliff import command from cliff import lister +from goal_tools import gitutils from goal_tools import governance LOG = logging.getLogger(__name__) @@ -19,19 +22,27 @@ def get_setup_config(repo_dir): return parser -def check_one(repo_base_dir, repo): +def applies_to_repo(repo): # Specs repositories aren't packaged if repo.endswith('-specs'): - return 'not needed' + return False # Charm repositories don't need to build wheels repo_base = repo.partition('/')[-1] if repo_base.startswith('charm-'): - return 'not needed' + return False # Puppet repos don't build wheels if repo_base.startswith('puppet-'): - return 'not needed' + return False # xstatic repos don't need wheels if repo_base.startswith('xstatic-'): + return False + if repo_base.endswith('-tempest-plugin'): + return False + return True + + +def check_one(repo_base_dir, repo): + if not applies_to_repo(repo): return 'not needed' repo_dir = os.path.join(os.path.expanduser(repo_base_dir), repo) if not os.path.exists(os.path.join(repo_dir, 'setup.cfg')): @@ -101,3 +112,93 @@ class WheelMissingUniversal(lister.Lister): ] return (columns, data) + + +COMMIT_MESSAGE = '''\ +build universal wheels + +By default setuptools produces a version-specific wheel file so +installation under other versions of Python require extra work at +install time. This change turns on "universal" wheel support, so that +the wheel file will be marked as supporting both Python 2 and 3. + +Signed-off-by: Doug Hellmann +''' + + +def fix_one(workdir, repo): + LOG.info('processing %s', repo) + repo_dir = os.path.join(workdir, repo) + gitutils.git(repo_dir, 'checkout', 'master') + gitutils.git(repo_dir, 'checkout', '-b', 'python3-first-wheels') + setup_file = os.path.join(repo_dir, 'setup.cfg') + with open(setup_file, 'r', encoding='utf-8') as f: + contents = f.read().rstrip() + contents = contents + '\n\n[wheel]\nuniversal = 1\n' + with open(setup_file, 'w', encoding='utf-8') as f: + f.write(contents) + gitutils.git(repo_dir, 'diff') + gitutils.git(repo_dir, 'add', 'setup.cfg') + gitutils.git(repo_dir, 'review', '-s') + gitutils.git(repo_dir, 'commit', '-m', COMMIT_MESSAGE) + gitutils.git(repo_dir, 'show') + + +class WheelFixMissingUniversal(command.Command): + "add the flag to build universal wheels" + + def get_parser(self, prog_name): + parser = super().get_parser(prog_name) + parser.add_argument( + '--project-list', + default=governance.PROJECTS_LIST, + help='URL for governance projects.yaml', + ) + parser.add_argument( + 'workdir', + help='working directory for output repositories', + ) + return parser + + def take_action(self, parsed_args): + gov_dat = governance.Governance(url=parsed_args.project_list) + repos = gov_dat.get_repos() + + teams_and_repos = sorted( + (gov_dat.get_repo_owner(r), r) + for r in repos + ) + + workdir = os.path.realpath(parsed_args.workdir) + + for team, r in teams_and_repos: + if team == 'Infrastructure': + LOG.info('skipping %s', r) + continue + if not applies_to_repo(r): + LOG.info('skipping %s', r) + continue + + team_dir = os.path.join(workdir, team).replace(' ', '-') + if not os.path.exists(team_dir): + LOG.info('creating %s', team_dir) + os.mkdir(team_dir) + + tracking_file = os.path.join(team_dir, 'master') + + gitutils.clone_repo(team_dir, r) + status = check_one(team_dir, r) + if status in ('OK', 'not needed'): + LOG.info('nothing to change for %s', r) + shutil.rmtree(os.path.join(team_dir, r)) + continue + + try: + fix_one(team_dir, r) + except Exception: + LOG.exception('failed to update {}'.format(r)) + continue + + LOG.info('adding %s to %s', r, tracking_file) + with open(tracking_file, 'a', encoding='utf-8') as f: + f.write('{}\n'.format(r)) diff --git a/setup.cfg b/setup.cfg index 3666295..77a4061 100644 --- a/setup.cfg +++ b/setup.cfg @@ -64,6 +64,7 @@ python3_first = tox missing = goal_tools.python3_first.toxsettings:ToxMissingPy3 tox fix = goal_tools.python3_first.toxsettings:ToxFixMissingPy3 wheel missing = goal_tools.python3_first.wheelsettings:WheelMissingUniversal + wheel fix = goal_tools.python3_first.wheelsettings:WheelFixMissingUniversal [wheel] universal = 1