From df387cafb077ed4211287361d1ff86acffb1fd86 Mon Sep 17 00:00:00 2001 From: Arx Cruz Date: Fri, 4 Jun 2021 14:55:42 +0200 Subject: [PATCH] Refactoring tripleo-repos This is a huge refactoring including: * Add cliff support for extended future commands * Move constants into constant file * Move template into templates file * Move exceptions into exceptions * Replace print statements with logging package At the end the code still remains the same, but requires some refactoring in the future Change-Id: I158f64a4bedbea293eb1ba5075ad264e847ff063 --- requirements.txt | 1 + setup.cfg | 2 + tripleo_repos/constants.py | 52 ++ tripleo_repos/exceptions.py | 23 + tripleo_repos/generate_repos.py | 467 +++++++++++++++ tripleo_repos/main.py | 538 +----------------- tripleo_repos/templates.py | 68 +++ .../{test_main.py => test_generate_repos.py} | 245 ++++---- 8 files changed, 783 insertions(+), 613 deletions(-) create mode 100644 tripleo_repos/constants.py create mode 100644 tripleo_repos/exceptions.py create mode 100644 tripleo_repos/generate_repos.py create mode 100644 tripleo_repos/templates.py rename tripleo_repos/tests/{test_main.py => test_generate_repos.py} (76%) diff --git a/requirements.txt b/requirements.txt index 2623409..ce3fde9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. +cliff>=3.8.0 pbr!=2.1.0,>=2.0.0 # Apache-2.0 requests>=2.10.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 6392b0d..c08bb10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,3 +28,5 @@ packages = [entry_points] console_scripts = tripleo-repos = tripleo_repos.main:main +tripleo_repos.cm = + generate = tripleo_repos.generate_repos:GenerateRepos diff --git a/tripleo_repos/constants.py b/tripleo_repos/constants.py new file mode 100644 index 0000000..7f1db91 --- /dev/null +++ b/tripleo_repos/constants.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# Copyright 2016 Red Hat, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + + +# Regexes +TITLE_RE = re.compile('\\[(.*)\\]') +NAME_RE = re.compile('name=(.+)') +PRIORITY_RE = re.compile('priority=\\d+') + + +# Packages to be included from delorean-current when using current-tripleo +INCLUDE_PKGS = ('includepkgs=instack,instack-undercloud,' + 'os-apply-config,os-collect-config,os-net-config,' + 'os-refresh-config,python*-tripleoclient,' + 'openstack-tripleo-*,openstack-puppet-modules,' + 'ansible-role-tripleo*,puppet-*,python*-tripleo-common,' + 'python*-paunch*,tripleo-ansible,ansible-config_template') + +# RHEL is only provided to licensed cloud providers via RHUI +DEFAULT_MIRROR_MAP = { + 'fedora': 'https://mirrors.fedoraproject.org', + 'centos': 'http://mirror.centos.org', + 'ubi': 'http://mirror.centos.org', + 'rhel': 'https://trunk.rdoproject.org', +} + +# unversioned fedora added for backwards compatibility +SUPPORTED_DISTROS = [ + ('centos', '7'), + ('centos', '8'), + ('fedora', ''), + ('rhel', '8'), + ('ubi', '8') # a subcase of the rhel distro +] + +DEFAULT_OUTPUT_PATH = '/etc/yum.repos.d' +DEFAULT_RDO_MIRROR = 'https://trunk.rdoproject.org' diff --git a/tripleo_repos/exceptions.py b/tripleo_repos/exceptions.py new file mode 100644 index 0000000..ec7f87d --- /dev/null +++ b/tripleo_repos/exceptions.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +# Copyright 2016 Red Hat, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class InvalidArguments(Exception): + pass + + +class NoRepoTitle(Exception): + pass diff --git a/tripleo_repos/generate_repos.py b/tripleo_repos/generate_repos.py new file mode 100644 index 0000000..56749c7 --- /dev/null +++ b/tripleo_repos/generate_repos.py @@ -0,0 +1,467 @@ +#!/usr/bin/python +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import re +import requests +import subprocess + +from cliff.command import Command + +import tripleo_repos.constants as C +import tripleo_repos.templates as T +import tripleo_repos.exceptions as e + + +class GenerateRepos(Command): + """Command to generate the repos""" + + log = logging.getLogger(__name__) + + def __init__(self, app, app_args, cmd_name=None): + super(GenerateRepos, self).__init__(app, app_args, cmd_name) + distro_info = self._get_distro() + self.distro_id = distro_info[0] + self.distro_major_version_id = distro_info[1] + self.distro_name = distro_info[2] + self.default_mirror = C.DEFAULT_MIRROR_MAP[self.distro_id] + + def take_action(self, parsed_args): + self.log.debug('Running GenerateRepos command') + if parsed_args.no_stream: + parsed_args.stream = False + parsed_args.old_mirror = self.default_mirror + + self._validate_args(parsed_args, self.distro_name) + base_path = self._get_base_path(parsed_args) + if parsed_args.distro in ['centos7']: + self._install_priorities() + self._remove_existing(parsed_args) + self._install_repos(parsed_args, base_path) + self._run_pkg_clean(parsed_args.distro) + + def get_parser(self, prog_name): + parser = super(GenerateRepos, self).get_parser(prog_name) + + distro = "{}{}".format(self.distro_id, self.distro_major_version_id) + + # Calculating arguments default from constants + distro_choices = ["".join(distro_pair) + for distro_pair in C.SUPPORTED_DISTROS] + + parser.add_argument('repos', metavar='REPO', nargs='+', + choices=['current', 'deps', 'current-tripleo', + 'current-tripleo-dev', 'ceph', 'opstools', + 'tripleo-ci-testing', + 'current-tripleo-rdo'], + help='A list of repos. Available repos: ' + '%(choices)s. The deps repo will always be ' + 'included when using current or ' + 'current-tripleo. current-tripleo-dev ' + 'downloads the current-tripleo, current, and ' + 'deps repos, but sets the current repo to ' + 'only be used for TripleO projects. ' + 'It also modifies each repo\'s priority so ' + 'packages are installed from the appropriate ' + 'location.') + parser.add_argument('-d', '--distro', + default=distro, + choices=distro_choices, + nargs='?', + help='Target distro with default detected at ' + 'runtime.' + ) + parser.add_argument('-b', '--branch', + default='master', + help='Target branch. Should be the lowercase ' + 'name of the OpenStack release. e.g. liberty') + parser.add_argument('-o', '--output-path', + default=C.DEFAULT_OUTPUT_PATH, + help='Directory in which to save the selected ' + 'repos.') + parser.add_argument('--mirror', + default=self.default_mirror, + help='Server from which to install base OS ' + 'packages. Default value is based on distro ' + 'param.') + parser.add_argument('--rdo-mirror', + default=C.DEFAULT_RDO_MIRROR, + help='Server from which to install RDO packages.') + + stream_group = parser.add_mutually_exclusive_group() + stream_group.add_argument('--stream', + action='store_true', + default=True, + help='Enable stream support for CentOS ' + 'repos') + stream_group.add_argument('--no-stream', + action='store_true', + default=False, + help='Disable stream support for CentOS ' + 'repos') + + return parser + + def _run_pkg_clean(self, distro): + pkg_mgr = 'yum' if distro == 'centos7' else 'dnf' + try: + subprocess.check_call([pkg_mgr, 'clean', 'metadata']) + except subprocess.CalledProcessError: + self.log.error('Failed to clean yum metadata.') + raise + + def _inject_mirrors(self, content, args): + """Replace any references to the default mirrors in repo content + + In some cases we want to use mirrors whose repo files still point to + the default servers. If the user specified to use the mirror, we want + to replace any such references with the mirror address. This function + handles that by using a regex to swap out the baseurl server. + """ + + content = re.sub('baseurl=%s' % C.DEFAULT_RDO_MIRROR, + 'baseurl=%s' % args.rdo_mirror, + content) + + if args.old_mirror: + content = re.sub('baseurl=%s' % args.old_mirror, + 'baseurl=%s' % args.mirror, + content) + + return content + + def _get_repo(self, path, args): + r = requests.get(path) + if r.status_code == 200: + return self._inject_mirrors(r.text, args) + else: + r.raise_for_status() + + def _write_repo(self, content, target, name=None): + if not name: + m = C.TITLE_RE.search(content) + if not m: + raise e.NoRepoTitle( + 'Could not find repo title in: \n%s' % content) + name = m.group(1) + # centos-8 dlrn repos have changed. repos per component + # are folded into a single repo. + if 'component' in name: + name = 'delorean' + filename = name + '.repo' + filename = os.path.join(target, filename) + with open(filename, 'w') as f: + f.write(content) + self.log.info('Installed repo %s to %s' % (name, filename)) + + def _change_priority(self, content, new_priority): + new_content = C.PRIORITY_RE.sub('priority=%d' % new_priority, content) + # This shouldn't happen, but let's be safe. + if not C.PRIORITY_RE.search(new_content): + new_content = [] + for line in content.split("\n"): + new_content.append(line) + if line.startswith('['): + new_content.append('priority=%d' % new_priority) + new_content = "\n".join(new_content) + return new_content + + def _create_ceph(self, args, release): + """Generate a Ceph repo file for release""" + centos_release = '7' if args.distro == 'centos7' else '8' + return T.CEPH_REPO_TEMPLATE % {'centos_release': centos_release, + 'ceph_release': release, + 'mirror': args.mirror} + + def _add_includepkgs(self, content): + new_content = [] + for line in content.split("\n"): + new_content.append(line) + if line.startswith('['): + new_content.append(C.INCLUDE_PKGS) + return "\n".join(new_content) + + # TODO: This need to be refactored + def _install_repos(self, args, base_path): + def install_deps(args, base_path): + content = self._get_repo(base_path + 'delorean-deps.repo', args) + self._write_repo(content, args.output_path) + for repo in args.repos: + if repo == 'current': + content = self._get_repo( + base_path + 'current/delorean.repo', args) + self._write_repo(content, args.output_path, name='delorean') + install_deps(args, base_path) + elif repo == 'deps': + install_deps(args, base_path) + elif repo == 'current-tripleo': + content = self._get_repo( + base_path + 'current-tripleo/delorean.repo', args) + self._write_repo(content, args.output_path) + install_deps(args, base_path) + elif repo == 'current-tripleo-dev': + content = self._get_repo( + base_path + 'delorean-deps.repo', args) + self._write_repo(content, args.output_path) + content = self._get_repo( + base_path + 'current-tripleo/delorean.repo', args) + content = C.TITLE_RE.sub('[\\1-current-tripleo]', content) + content = C.NAME_RE.sub('name=\\1-current-tripleo', content) + # We need to twiddle priorities since we're mixing multiple + # repos that are generated with the same priority. + content = self._change_priority(content, 20) + self._write_repo(content, args.output_path, + name='delorean-current-tripleo') + content = self._get_repo( + base_path + 'current/delorean.repo', args) + content = self._add_includepkgs(content) + content = self._change_priority(content, 10) + self._write_repo(content, args.output_path, name='delorean') + elif repo == 'tripleo-ci-testing': + content = self._get_repo( + base_path + 'tripleo-ci-testing/delorean.repo', args) + self._write_repo(content, args.output_path) + install_deps(args, base_path) + elif repo == 'current-tripleo-rdo': + content = self._get_repo( + base_path + 'current-tripleo-rdo/delorean.repo', args) + self._write_repo(content, args.output_path) + install_deps(args, base_path) + elif repo == 'ceph': + if args.branch in ['liberty', 'mitaka']: + content = self._create_ceph(args, 'hammer') + elif args.branch in ['newton', 'ocata', 'pike']: + content = self._create_ceph(args, 'jewel') + elif args.branch in ['queens', 'rocky']: + content = self._create_ceph(args, 'luminous') + elif args.branch in ['stein', 'train', 'ussuri', 'victoria']: + content = self._create_ceph(args, 'nautilus') + else: + content = self._create_ceph(args, 'pacific') + self._write_repo(content, args.output_path) + elif repo == 'opstools': + content = T.OPSTOOLS_REPO_TEMPLATE % {'mirror': args.mirror} + self._write_repo(content, args.output_path) + else: + raise e.InvalidArguments('Invalid repo "%s" specified' % repo) + + distro = args.distro + # CentOS-8 AppStream is required for UBI-8 + if distro == 'ubi8': + if not os.path.exists("/etc/distro.repos.d"): + self.log.warning('For UBI it is recommended to create ' + '/etc/distro.repos.d and rerun!') + dp_exists = False + else: + dp_exists = True + if args.output_path == C.DEFAULT_OUTPUT_PATH and dp_exists: + distro_path = "/etc/distro.repos.d" + else: + distro_path = args.output_path + # TODO: Remove it once bugs are fixed + # Add extra options to APPSTREAM_REPO_TEMPLATE because of + # rhbz/1961558 and lpbz/1929634 + extra = '' + if args.branch in ['train', 'ussuri', 'victoria']: + extra = 'exclude=edk2-ovmf-20200602gitca407c7246bf-5*' + content = T.APPSTREAM_REPO_TEMPLATE % {'mirror': args.mirror, + 'extra': extra} + self._write_repo(content, distro_path) + content = T.BASE_REPO_TEMPLATE % {'mirror': args.mirror} + self._write_repo(content, distro_path) + distro = 'centos8' # switch it to continue as centos8 distro + + # HA, Powertools are required for CentOS-8 + if distro == 'centos8': + stream = '8' + if args.stream and not args.no_stream: + stream = stream + '-stream' + content = T.HIGHAVAILABILITY_REPO_TEMPLATE % { + 'mirror': args.mirror, 'stream': stream} + self._write_repo(content, args.output_path) + content = T.POWERTOOLS_REPO_TEMPLATE % {'mirror': args.mirror, + 'stream': stream} + self._write_repo(content, args.output_path) + + def _install_priorities(self): + try: + subprocess.check_call(['yum', 'install', '-y', + 'yum-plugin-priorities']) + except subprocess.CalledProcessError as e: + self.log.error('Failed to install yum-plugin-priorities\n%s\n%s' % + (e.cmd, e.output)) + raise + + def _get_distro(self): + """Get distro info from os-release + + returns: distro_id, distro_major_version_id, distro_name + """ + + output = subprocess.Popen( + 'source /etc/os-release && echo -e -n "$ID\n$VERSION_ID\n$NAME"', + shell=True, + stdout=subprocess.PIPE, + stderr=open(os.devnull, 'w'), + executable='/bin/bash', + universal_newlines=True).communicate() + + # distro_id and distro_version_id will always be at least an + # empty string + distro_id, distro_version_id, distro_name = output[0].split('\n') + + # if distro_version_id is empty string the major version will be empty + # string too + distro_major_version_id = distro_version_id.split('.')[0] + + # check if that is UBI subcase? + if os.path.exists('/etc/yum.repos.d/ubi.repo'): + distro_id = 'ubi' + + if (distro_id, distro_major_version_id) not in C.SUPPORTED_DISTROS: + self.log.warning( + "Unsupported platform '{}{}' detected by tripleo-repos," + " centos7 will be used unless you use CLI param to change it." + "".format(distro_id, distro_major_version_id)) + distro_id = 'centos' + distro_major_version_id = '7' + + if distro_id == 'ubi': + self.log.warning( + "Centos{} Base and AppStream will be installed for " + "this UBI distro".format(distro_major_version_id)) + + return distro_id, distro_major_version_id, distro_name + + def _remove_existing(self, args): + """Remove any delorean* or opstools repos that already exist""" + if args.distro == 'ubi8': + regex = '^(BaseOS|AppStream|delorean|tripleo-centos-' \ + '(opstools|ceph|highavailability|powertools)).*.repo' + else: + regex = '^(delorean|tripleo-centos-' \ + '(opstools|ceph|highavailability|powertools)).*.repo' + pattern = re.compile(regex) + if os.path.exists("/etc/distro.repos.d"): + paths = set( + os.listdir(args.output_path) + os.listdir( + "/etc/distro.repos.d")) + else: + paths = os.listdir(args.output_path) + for f in paths: + if pattern.match(f): + filename = os.path.join(args.output_path, f) + if os.path.exists(filename): + os.remove(filename) + self.log.info('Removed old repo "%s"' % filename) + filename = os.path.join("/etc/distro.repos.d", f) + if os.path.exists(filename): + os.remove(filename) + self.log.info('Removed old repo "%s"' % filename) + + def _get_base_path(self, args): + if args.distro == 'ubi8': + # there are no base paths for UBI that work well + distro = 'centos8' + else: + distro = args.distro + + # The mirror url with /$DISTRO$VERSION path for master branch is + # deprecated. + # The default for rdo mirrors is $DISTRO$VERSION-$BRANCH + # it should work for every (distro, branch) pair that + # makes sense + # Any exception should be corrected at source, not here. + distro_branch = '%s-%s' % (distro, args.branch) + return '%s/%s/' % (args.rdo_mirror, distro_branch) + + # Validation functions + + def _validate_args(self, args, distro_name): + self._validate_current_tripleo(args.repos) + self._validate_distro_repos(args) + self._validate_tripleo_ci_testing(args.repos) + self._validate_distro_stream(args, distro_name) + + def _validate_distro_repos(self, args): + """Validate requested repos are valid for the distro""" + valid_repos = [] + if 'fedora' in args.distro: + valid_repos = ['current', 'current-tripleo', 'ceph', 'deps', + 'tripleo-ci-testing'] + elif args.distro in ['centos7', 'centos8', 'rhel8', 'ubi8']: + valid_repos = ['ceph', 'current', 'current-tripleo', + 'current-tripleo-dev', 'deps', 'tripleo-ci-testing', + 'opstools', 'current-tripleo-rdo'] + invalid_repos = [x for x in args.repos if x not in valid_repos] + if len(invalid_repos) > 0: + raise e.InvalidArguments('{} repo(s) are not valid for {}. Valid ' + 'repos are: {}'.format(invalid_repos, + args.distro, + valid_repos)) + return True + + def _validate_tripleo_ci_testing(self, repos): + """Validate tripleo-ci-testing + + With tripleo-ci-testing for repo (currently only periodic container + build) no other repos expected except optionally deps|ceph|opstools + which is enabled regardless. + """ + if 'tripleo-ci-testing' in repos and len(repos) > 1: + if 'deps' in repos or 'ceph' in repos or 'opstools' in repos: + return True + else: + raise e.InvalidArguments('Cannot use tripleo-ci-testing at the' + ' same time as other repos, except ' + 'deps|ceph|opstools.') + return True + + def _validate_distro_stream(self, args, distro_name): + """Validate stream related args vs host + + Fails if stream is to be used but the host isn't a stream OS or + vice versa + """ + is_stream = args.stream and not args.no_stream + if is_stream and 'stream' not in distro_name.lower(): + raise e.InvalidArguments('--stream provided, but OS is not the ' + 'Stream version. Please ensure the host ' + 'is Stream.') + elif not is_stream and 'stream' in distro_name.lower(): + raise e.InvalidArguments('--no-stream provided, but OS is the ' + 'Stream version. Please ensure the host ' + 'is not the Stream version.') + return True + + def _validate_current_tripleo(self, repos): + """Validate current usage + + current and current-tripleo cannot be specified with each other and + current-tripleo-dev is a mix of current, current-tripleo and deps + so they should not be specified on the command line with each other. + """ + if 'current-tripleo' in repos and 'current' in repos: + raise e.InvalidArguments( + 'Cannot use current and current-tripleo at the same time.') + if 'current-tripleo-dev' not in repos: + return True + if 'current' in repos or 'current-tripleo' in repos or 'deps' in repos: + raise e.InvalidArguments( + 'current-tripleo-dev should not be used with any other ' + 'RDO Trunk repos.') + return True diff --git a/tripleo_repos/main.py b/tripleo_repos/main.py index 6f299b7..b9d59b4 100755 --- a/tripleo_repos/main.py +++ b/tripleo_repos/main.py @@ -14,531 +14,41 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function -import argparse -import os -import re -import subprocess import sys -import requests +from cliff.app import App +from cliff.commandmanager import CommandManager -TITLE_RE = re.compile('\\[(.*)\\]') -NAME_RE = re.compile('name=(.+)') -PRIORITY_RE = re.compile('priority=\\d+') -# Packages to be included from delorean-current when using current-tripleo -INCLUDE_PKGS = ('includepkgs=instack,instack-undercloud,' - 'os-apply-config,os-collect-config,os-net-config,' - 'os-refresh-config,python*-tripleoclient,' - 'openstack-tripleo-*,openstack-puppet-modules,' - 'ansible-role-tripleo*,puppet-*,python*-tripleo-common,' - 'python*-paunch*,tripleo-ansible,ansible-config_template') -DEFAULT_OUTPUT_PATH = '/etc/yum.repos.d' -DEFAULT_RDO_MIRROR = 'https://trunk.rdoproject.org' +class TripleoReposApp(App): -# RHEL is only provided to licensed cloud providers via RHUI -DEFAULT_MIRROR_MAP = { - 'fedora': 'https://mirrors.fedoraproject.org', - 'centos': 'http://mirror.centos.org', - 'ubi': 'http://mirror.centos.org', - 'rhel': 'https://trunk.rdoproject.org', -} -CEPH_REPO_TEMPLATE = ''' -[tripleo-centos-ceph-%(ceph_release)s] -name=tripleo-centos-ceph-%(ceph_release)s -baseurl=%(mirror)s/centos/%(centos_release)s/storage/$basearch/ceph-%(ceph_release)s/ -gpgcheck=0 -enabled=1 -''' -OPSTOOLS_REPO_TEMPLATE = ''' -[tripleo-centos-opstools] -name=tripleo-centos-opstools -baseurl=%(mirror)s/centos/7/opstools/$basearch/ -gpgcheck=0 -enabled=1 -''' -# centos-8 only -HIGHAVAILABILITY_REPO_TEMPLATE = ''' -[tripleo-centos-highavailability] -name=tripleo-centos-highavailability -baseurl=%(mirror)s/centos/%(stream)s/HighAvailability/$basearch/os/ -gpgcheck=0 -enabled=1 -''' -# centos-8 only -POWERTOOLS_REPO_TEMPLATE = ''' -[tripleo-centos-powertools] -name=tripleo-centos-powertools -baseurl=%(mirror)s/centos/%(stream)s/PowerTools/$basearch/os/ -gpgcheck=0 -enabled=1 -''' -# ubi-8 only -APPSTREAM_REPO_TEMPLATE = ''' -[AppStream] -name=CentOS-$releasever - AppStream -baseurl=%(mirror)s/centos/$releasever/AppStream/$basearch/os/ -gpgcheck=0 -enabled=1 -%(extra)s -''' -BASE_REPO_TEMPLATE = ''' -[BaseOS] -name=CentOS-$releasever - Base -baseurl=%(mirror)s/centos/$releasever/BaseOS/$basearch/os/ -gpgcheck=0 -enabled=1 -''' + def __init__(self): + super(TripleoReposApp, self).__init__( + description='Tripleo repos tool', + version='2.0', + command_manager=CommandManager('tripleo_repos.cm'), + deferred_help=True) + def initialize_app(self, argv): + self.LOG.debug('Initializing tripleo-repos tool') -# unversioned fedora added for backwards compatibility -SUPPORTED_DISTROS = [ - ('centos', '7'), - ('centos', '8'), - ('fedora', ''), - ('rhel', '8'), - ('ubi', '8') # a subcase of the rhel distro -] + def prepare_to_run_command(self, cmd): + self.LOG.debug('prepare_to_run_command %s', cmd.__class__.__name__) + def clean_up(self, cmd, result, err): + self.LOG.debug('clean_up %s', cmd.__class__.__name__) + if err: + self.LOG.debug('Error: %s', err) -class InvalidArguments(Exception): - pass +def main(argv=sys.argv[1:]): + tripleo_app = TripleoReposApp() -class NoRepoTitle(Exception): - pass - - -def _get_distro(): - """Get distro info from os-release - - returns: distro_id, distro_major_version_id, distro_name - """ - output = subprocess.Popen( - 'source /etc/os-release && echo -e -n "$ID\n$VERSION_ID\n$NAME"', - shell=True, - stdout=subprocess.PIPE, - stderr=open(os.devnull, 'w'), - executable='/bin/bash', - universal_newlines=True).communicate() - - # distro_id and distro_version_id will always be at least an empty string - distro_id, distro_version_id, distro_name = output[0].split('\n') - - # if distro_version_id is empty string the major version will be empty - # string too - distro_major_version_id = distro_version_id.split('.')[0] - - # check if that is UBI subcase? - if os.path.exists('/etc/yum.repos.d/ubi.repo'): - distro_id = 'ubi' - - if (distro_id, distro_major_version_id) not in SUPPORTED_DISTROS: - print( - "WARNING: Unsupported platform '{}{}' detected by tripleo-repos," - " centos7 will be used unless you use CLI param to change it." - "".format(distro_id, distro_major_version_id), file=sys.stderr) - distro_id = 'centos' - distro_major_version_id = '7' - - if distro_id == 'ubi': - print( - "WARNING: Centos{} Base and AppStream will be installed for " - "this UBI distro".format(distro_major_version_id)) - - return distro_id, distro_major_version_id, distro_name - - -def _parse_args(distro_id, distro_major_version_id): - - distro = "{}{}".format(distro_id, distro_major_version_id) - - # Calculating arguments default from constants - default_mirror = DEFAULT_MIRROR_MAP[distro_id] - distro_choices = ["".join(distro_pair) - for distro_pair in SUPPORTED_DISTROS] - - parser = argparse.ArgumentParser( - description='Download and install repos necessary for TripleO. Note ' - 'that some of these repos require yum-plugin-priorities, ' - 'so that will also be installed.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('repos', metavar='REPO', nargs='+', - choices=['current', 'deps', 'current-tripleo', - 'current-tripleo-dev', 'ceph', 'opstools', - 'tripleo-ci-testing', 'current-tripleo-rdo'], - help='A list of repos. Available repos: ' - '%(choices)s. The deps repo will always be ' - 'included when using current or ' - 'current-tripleo. current-tripleo-dev ' - 'downloads the current-tripleo, current, and ' - 'deps repos, but sets the current repo to only ' - 'be used for TripleO projects. It also modifies ' - 'each repo\'s priority so packages are installed ' - 'from the appropriate location.') - parser.add_argument('-d', '--distro', - default=distro, - choices=distro_choices, - nargs='?', - help='Target distro with default detected at runtime. ' - ) - parser.add_argument('-b', '--branch', - default='master', - help='Target branch. Should be the lowercase name of ' - 'the OpenStack release. e.g. liberty') - parser.add_argument('-o', '--output-path', - default=DEFAULT_OUTPUT_PATH, - help='Directory in which to save the selected repos.') - parser.add_argument('--mirror', - default=default_mirror, - help='Server from which to install base OS packages. ' - 'Default value is based on distro param.') - parser.add_argument('--rdo-mirror', - default=DEFAULT_RDO_MIRROR, - help='Server from which to install RDO packages.') - stream_group = parser.add_mutually_exclusive_group() - stream_group.add_argument('--stream', - action='store_true', - default=True, - help='Enable stream support for CentOS repos') - stream_group.add_argument('--no-stream', - action='store_true', - default=False, - help='Disable stream support for CentOS repos') - - args = parser.parse_args() - if args.no_stream: - args.stream = False - args.old_mirror = default_mirror - - return args - - -def _get_repo(path, args): - r = requests.get(path) - if r.status_code == 200: - return _inject_mirrors(r.text, args) - else: - r.raise_for_status() - - -def _write_repo(content, target, name=None): - if not name: - m = TITLE_RE.search(content) - if not m: - raise NoRepoTitle('Could not find repo title in: \n%s' % content) - name = m.group(1) - # centos-8 dlrn repos have changed. repos per component - # are folded into a single repo. - if 'component' in name: - name = 'delorean' - filename = name + '.repo' - filename = os.path.join(target, filename) - with open(filename, 'w') as f: - f.write(content) - print('Installed repo %s to %s' % (name, filename)) - - -def _validate_distro_repos(args): - """Validate requested repos are valid for the distro""" - valid_repos = [] - if 'fedora' in args.distro: - valid_repos = ['current', 'current-tripleo', 'ceph', 'deps', - 'tripleo-ci-testing'] - elif args.distro in ['centos7', 'centos8', 'rhel8', 'ubi8']: - valid_repos = ['ceph', 'current', 'current-tripleo', - 'current-tripleo-dev', 'deps', 'tripleo-ci-testing', - 'opstools', 'current-tripleo-rdo'] - invalid_repos = [x for x in args.repos if x not in valid_repos] - if len(invalid_repos) > 0: - raise InvalidArguments('{} repo(s) are not valid for {}. Valid repos ' - 'are: {}'.format(invalid_repos, args.distro, - valid_repos)) - return True - - -def _validate_current_tripleo(repos): - """Validate current usage - - current and current-tripleo cannot be specified with each other and - current-tripleo-dev is a mix of current, current-tripleo and deps - so they should not be specified on the command line with each other. - """ - if 'current-tripleo' in repos and 'current' in repos: - raise InvalidArguments('Cannot use current and current-tripleo at the ' - 'same time.') - if 'current-tripleo-dev' not in repos: - return True - if 'current' in repos or 'current-tripleo' in repos or 'deps' in repos: - raise InvalidArguments('current-tripleo-dev should not be used with ' - 'any other RDO Trunk repos.') - return True - - -def _validate_tripleo_ci_testing(repos): - """Validate tripleo-ci-testing - - With tripleo-ci-testing for repo (currently only periodic container build) - no other repos expected except optionally deps|ceph|opstools - which is enabled regardless. - """ - if 'tripleo-ci-testing' in repos and len(repos) > 1: - if 'deps' in repos or 'ceph' in repos or 'opstools' in repos: - return True - else: - raise InvalidArguments('Cannot use tripleo-ci-testing at the ' - 'same time as other repos, except ' - 'deps|ceph|opstools.') - return True - - -def _validate_distro_stream(args, distro_name): - """Validate stream related args vs host - - Fails if stream is to be used but the host isn't a stream OS or vice versa - """ - is_stream = args.stream and not args.no_stream - if is_stream and 'stream' not in distro_name.lower(): - raise InvalidArguments('--stream provided, but OS is not the Stream ' - 'version. Please ensure the host is Stream.') - elif not is_stream and 'stream' in distro_name.lower(): - raise InvalidArguments('--no-stream provided, but OS is the Stream ' - 'version. Please ensure the host is not the ' - 'Stream version.') - return True - - -def _validate_args(args, distro_name): - _validate_current_tripleo(args.repos) - _validate_distro_repos(args) - _validate_tripleo_ci_testing(args.repos) - _validate_distro_stream(args, distro_name) - - -def _remove_existing(args): - """Remove any delorean* or opstools repos that already exist""" - if args.distro == 'ubi8': - regex = '^(BaseOS|AppStream|delorean|tripleo-centos-' \ - '(opstools|ceph|highavailability|powertools)).*.repo' - else: - regex = '^(delorean|tripleo-centos-' \ - '(opstools|ceph|highavailability|powertools)).*.repo' - pattern = re.compile(regex) - if os.path.exists("/etc/distro.repos.d"): - paths = set( - os.listdir(args.output_path) + os.listdir("/etc/distro.repos.d")) - else: - paths = os.listdir(args.output_path) - for f in paths: - if pattern.match(f): - filename = os.path.join(args.output_path, f) - if os.path.exists(filename): - os.remove(filename) - print('Removed old repo "%s"' % filename) - filename = os.path.join("/etc/distro.repos.d", f) - if os.path.exists(filename): - os.remove(filename) - print('Removed old repo "%s"' % filename) - - -def _get_base_path(args): - if args.distro == 'ubi8': - distro = 'centos8' # there are no base paths for UBI that work well - else: - distro = args.distro - - # The mirror url with /$DISTRO$VERSION path for master branch is - # deprecated. - # The default for rdo mirrors is $DISTRO$VERSION-$BRANCH - # it should work for every (distro, branch) pair that - # makes sense - # Any exception should be corrected at source, not here. - distro_branch = '%s-%s' % (distro, args.branch) - return '%s/%s/' % (args.rdo_mirror, distro_branch) - - -def _install_priorities(): - try: - subprocess.check_call(['yum', 'install', '-y', - 'yum-plugin-priorities']) - except subprocess.CalledProcessError as e: - print('ERROR: Failed to install yum-plugin-priorities\n%s\n%s' % - (e.cmd, e.output)) - raise - - -def _create_ceph(args, release): - """Generate a Ceph repo file for release""" - centos_release = '7' if args.distro == 'centos7' else '8' - return CEPH_REPO_TEMPLATE % {'centos_release': centos_release, - 'ceph_release': release, - 'mirror': args.mirror} - - -def _change_priority(content, new_priority): - new_content = PRIORITY_RE.sub('priority=%d' % new_priority, content) - # This shouldn't happen, but let's be safe. - if not PRIORITY_RE.search(new_content): - new_content = [] - for line in content.split("\n"): - new_content.append(line) - if line.startswith('['): - new_content.append('priority=%d' % new_priority) - new_content = "\n".join(new_content) - return new_content - - -def _add_includepkgs(content): - new_content = [] - for line in content.split("\n"): - new_content.append(line) - if line.startswith('['): - new_content.append(INCLUDE_PKGS) - return "\n".join(new_content) - - -def _inject_mirrors(content, args): - """Replace any references to the default mirrors in repo content - - In some cases we want to use mirrors whose repo files still point to the - default servers. If the user specified to use the mirror, we want to - replace any such references with the mirror address. This function - handles that by using a regex to swap out the baseurl server. - """ - - content = re.sub('baseurl=%s' % DEFAULT_RDO_MIRROR, - 'baseurl=%s' % args.rdo_mirror, - content) - - if args.old_mirror: - content = re.sub('baseurl=%s' % args.old_mirror, - 'baseurl=%s' % args.mirror, - content) - - return content - - -def _install_repos(args, base_path): - def install_deps(args, base_path): - content = _get_repo(base_path + 'delorean-deps.repo', args) - _write_repo(content, args.output_path) - - for repo in args.repos: - if repo == 'current': - content = _get_repo(base_path + 'current/delorean.repo', args) - _write_repo(content, args.output_path, name='delorean') - install_deps(args, base_path) - elif repo == 'deps': - install_deps(args, base_path) - elif repo == 'current-tripleo': - content = _get_repo(base_path + 'current-tripleo/delorean.repo', - args) - _write_repo(content, args.output_path) - install_deps(args, base_path) - elif repo == 'current-tripleo-dev': - content = _get_repo(base_path + 'delorean-deps.repo', args) - _write_repo(content, args.output_path) - content = _get_repo(base_path + 'current-tripleo/delorean.repo', - args) - content = TITLE_RE.sub('[\\1-current-tripleo]', content) - content = NAME_RE.sub('name=\\1-current-tripleo', content) - # We need to twiddle priorities since we're mixing multiple repos - # that are generated with the same priority. - content = _change_priority(content, 20) - _write_repo(content, args.output_path, - name='delorean-current-tripleo') - content = _get_repo(base_path + 'current/delorean.repo', args) - content = _add_includepkgs(content) - content = _change_priority(content, 10) - _write_repo(content, args.output_path, name='delorean') - elif repo == 'tripleo-ci-testing': - content = _get_repo(base_path + 'tripleo-ci-testing/delorean.repo', - args) - _write_repo(content, args.output_path) - install_deps(args, base_path) - elif repo == 'current-tripleo-rdo': - content = _get_repo( - base_path + 'current-tripleo-rdo/delorean.repo', args) - _write_repo(content, args.output_path) - install_deps(args, base_path) - elif repo == 'ceph': - if args.branch in ['liberty', 'mitaka']: - content = _create_ceph(args, 'hammer') - elif args.branch in ['newton', 'ocata', 'pike']: - content = _create_ceph(args, 'jewel') - elif args.branch in ['queens', 'rocky']: - content = _create_ceph(args, 'luminous') - elif args.branch in ['stein', 'train', 'ussuri', 'victoria']: - content = _create_ceph(args, 'nautilus') - else: - content = _create_ceph(args, 'pacific') - _write_repo(content, args.output_path) - elif repo == 'opstools': - content = OPSTOOLS_REPO_TEMPLATE % {'mirror': args.mirror} - _write_repo(content, args.output_path) - else: - raise InvalidArguments('Invalid repo "%s" specified' % repo) - - distro = args.distro - # CentOS-8 AppStream is required for UBI-8 - if distro == 'ubi8': - if not os.path.exists("/etc/distro.repos.d"): - print('WARNING: For UBI it is recommended to create ' - '/etc/distro.repos.d and rerun!') - dp_exists = False - else: - dp_exists = True - if args.output_path == DEFAULT_OUTPUT_PATH and dp_exists: - distro_path = "/etc/distro.repos.d" - else: - distro_path = args.output_path - # TODO: Remove it once bugs are fixed - # Add extra options to APPSTREAM_REPO_TEMPLATE because of - # rhbz/1961558 and lpbz/1929634 - extra = '' - if args.branch in ['train', 'ussuri', 'victoria']: - extra = 'exclude=edk2-ovmf-20200602gitca407c7246bf-5*' - content = APPSTREAM_REPO_TEMPLATE % {'mirror': args.mirror, - 'extra': extra} - _write_repo(content, distro_path) - content = BASE_REPO_TEMPLATE % {'mirror': args.mirror} - _write_repo(content, distro_path) - distro = 'centos8' # switch it to continue as centos8 distro - - # HA, Powertools are required for CentOS-8 - if distro == 'centos8': - stream = '8' - if args.stream and not args.no_stream: - stream = stream + '-stream' - content = HIGHAVAILABILITY_REPO_TEMPLATE % {'mirror': args.mirror, - 'stream': stream} - _write_repo(content, args.output_path) - content = POWERTOOLS_REPO_TEMPLATE % {'mirror': args.mirror, - 'stream': stream} - _write_repo(content, args.output_path) - - -def _run_pkg_clean(distro): - pkg_mgr = 'yum' if distro == 'centos7' else 'dnf' - try: - subprocess.check_call([pkg_mgr, 'clean', 'metadata']) - except subprocess.CalledProcessError: - print('ERROR: Failed to clean yum metadata.') - raise - - -def main(): - distro_id, distro_major_version_id, distro_name = _get_distro() - args = _parse_args(distro_id, distro_major_version_id) - _validate_args(args, distro_name) - base_path = _get_base_path(args) - if args.distro in ['centos7']: - _install_priorities() - _remove_existing(args) - _install_repos(args, base_path) - _run_pkg_clean(args.distro) + # Hack to keep compatibility for now + if 'generate' not in argv: + argv.insert(0, 'generate') + return tripleo_app.run(argv) if __name__ == '__main__': - main() + sys.exit(main(sys.argv[1:])) diff --git a/tripleo_repos/templates.py b/tripleo_repos/templates.py new file mode 100644 index 0000000..1952a52 --- /dev/null +++ b/tripleo_repos/templates.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +# Copyright 2016 Red Hat, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +CEPH_REPO_TEMPLATE = ''' +[tripleo-centos-ceph-%(ceph_release)s] +name=tripleo-centos-ceph-%(ceph_release)s +baseurl=%(mirror)s/centos/%(centos_release)s/storage/$basearch/ceph-%(ceph_release)s/ +gpgcheck=0 +enabled=1 +''' + +OPSTOOLS_REPO_TEMPLATE = ''' +[tripleo-centos-opstools] +name=tripleo-centos-opstools +baseurl=%(mirror)s/centos/7/opstools/$basearch/ +gpgcheck=0 +enabled=1 +''' + +# centos-8 only +HIGHAVAILABILITY_REPO_TEMPLATE = ''' +[tripleo-centos-highavailability] +name=tripleo-centos-highavailability +baseurl=%(mirror)s/centos/%(stream)s/HighAvailability/$basearch/os/ +gpgcheck=0 +enabled=1 +''' + +# centos-8 only +POWERTOOLS_REPO_TEMPLATE = ''' +[tripleo-centos-powertools] +name=tripleo-centos-powertools +baseurl=%(mirror)s/centos/%(stream)s/PowerTools/$basearch/os/ +gpgcheck=0 +enabled=1 +''' + +# ubi-8 only +APPSTREAM_REPO_TEMPLATE = ''' +[AppStream] +name=CentOS-$releasever - AppStream +baseurl=%(mirror)s/centos/$releasever/AppStream/$basearch/os/ +gpgcheck=0 +enabled=1 +%(extra)s +''' + +BASE_REPO_TEMPLATE = ''' +[BaseOS] +name=CentOS-$releasever - Base +baseurl=%(mirror)s/centos/$releasever/BaseOS/$basearch/os/ +gpgcheck=0 +enabled=1 +''' diff --git a/tripleo_repos/tests/test_main.py b/tripleo_repos/tests/test_generate_repos.py similarity index 76% rename from tripleo_repos/tests/test_main.py rename to tripleo_repos/tests/test_generate_repos.py index 9c766bb..b893165 100644 --- a/tripleo_repos/tests/test_main.py +++ b/tripleo_repos/tests/test_generate_repos.py @@ -19,26 +19,32 @@ from unittest import mock import ddt import testtools -from tripleo_repos import main +from tripleo_repos.generate_repos import GenerateRepos + +import tripleo_repos.exceptions as E +import tripleo_repos.constants as C @ddt.ddt class TestTripleORepos(testtools.TestCase): - @mock.patch('tripleo_repos.main._get_distro') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_distro') @mock.patch('sys.argv', ['tripleo-repos', 'current', '-d', 'centos7']) - @mock.patch('tripleo_repos.main._run_pkg_clean') - @mock.patch('tripleo_repos.main._validate_args') - @mock.patch('tripleo_repos.main._get_base_path') - @mock.patch('tripleo_repos.main._install_priorities') - @mock.patch('tripleo_repos.main._remove_existing') - @mock.patch('tripleo_repos.main._install_repos') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._run_pkg_clean') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._validate_args') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_base_path') + @mock.patch( + 'tripleo_repos.generate_repos.GenerateRepos._install_priorities') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._remove_existing') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._install_repos') def test_main(self, mock_install, mock_remove, mock_ip, mock_gbp, mock_validate, mock_clean, mock_distro): mock_distro.return_value = ('centos', '8', 'CentOS 8') - args = main._parse_args('centos', '8') + + self.cmd = GenerateRepos(None, None) + args = self.cmd.get_parser('NAME').parse_args() mock_path = mock.Mock() mock_gbp.return_value = mock_path - main.main() + self.cmd.run(args) mock_validate.assert_called_once_with(args, 'CentOS 8') mock_gbp.assert_called_once_with(args) mock_ip.assert_called_once_with() @@ -46,21 +52,23 @@ class TestTripleORepos(testtools.TestCase): mock_install.assert_called_once_with(args, mock_path) mock_clean.assert_called_once_with('centos7') - @mock.patch('tripleo_repos.main._get_distro') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_distro') @mock.patch('sys.argv', ['tripleo-repos', 'current', '-d', 'fedora']) - @mock.patch('tripleo_repos.main._run_pkg_clean') - @mock.patch('tripleo_repos.main._validate_args') - @mock.patch('tripleo_repos.main._get_base_path') - @mock.patch('tripleo_repos.main._install_priorities') - @mock.patch('tripleo_repos.main._remove_existing') - @mock.patch('tripleo_repos.main._install_repos') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._run_pkg_clean') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._validate_args') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_base_path') + @mock.patch( + 'tripleo_repos.generate_repos.GenerateRepos._install_priorities') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._remove_existing') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._install_repos') def test_main_fedora(self, mock_install, mock_remove, mock_ip, mock_gbp, mock_validate, mock_clean, mock_distro): mock_distro.return_value = ('centos', '8', 'CentOS 8') - args = main._parse_args('centos', '8') + self.cmd = GenerateRepos(None, None) + args = self.cmd.get_parser('NAME').parse_args() mock_path = mock.Mock() mock_gbp.return_value = mock_path - main.main() + self.cmd.run(args) mock_validate.assert_called_once_with(args, 'CentOS 8') mock_gbp.assert_called_once_with(args) assert not mock_ip.called, '_install_priorities should no tbe called' @@ -77,7 +85,8 @@ class TestTripleORepos(testtools.TestCase): fake_addr = 'http://lone/pine/mall' args = mock.Mock() args.distro = 'centos' - content = main._get_repo(fake_addr, args) + cmd = GenerateRepos(None, None) + content = cmd._get_repo(fake_addr, args) self.assertEqual('88MPH', content) mock_get.assert_called_once_with(fake_addr) @@ -87,7 +96,8 @@ class TestTripleORepos(testtools.TestCase): mock_response.status_code = 404 mock_get.return_value = mock_response fake_addr = 'http://twin/pines/mall' - main._get_repo(fake_addr, mock.Mock()) + cmd = GenerateRepos(None, None) + cmd._get_repo(fake_addr, mock.Mock()) mock_get.assert_called_once_with(fake_addr) mock_response.raise_for_status.assert_called_once_with() @@ -104,7 +114,8 @@ class TestTripleORepos(testtools.TestCase): mock_listdir.return_value = fake_list mock_args = mock.Mock() mock_args.output_path = '/etc/yum.repos.d' - main._remove_existing(mock_args) + cmd = GenerateRepos(None, None) + cmd._remove_existing(mock_args) self.assertIn(mock.call('/etc/yum.repos.d/delorean.repo'), mock_remove.mock_calls) self.assertIn(mock.call('/etc/yum.repos.d/' @@ -127,7 +138,8 @@ class TestTripleORepos(testtools.TestCase): args.branch = 'master' args.distro = 'centos7' args.rdo_mirror = 'http://trunk.rdoproject.org' - path = main._get_base_path(args) + cmd = GenerateRepos(None, None) + path = cmd._get_base_path(args) self.assertEqual('http://trunk.rdoproject.org/centos7-master/', path) def test_get_base_path_fedora(self): @@ -135,30 +147,34 @@ class TestTripleORepos(testtools.TestCase): args.branch = 'master' args.distro = 'fedora' args.rdo_mirror = 'http://trunk.rdoproject.org' - path = main._get_base_path(args) + cmd = GenerateRepos(None, None) + path = cmd._get_base_path(args) self.assertEqual('http://trunk.rdoproject.org/fedora-master/', path) @mock.patch('subprocess.check_call') def test_install_priorities(self, mock_check_call): - main._install_priorities() + cmd = GenerateRepos(None, None) + cmd._install_priorities() mock_check_call.assert_called_once_with(['yum', 'install', '-y', 'yum-plugin-priorities']) @mock.patch('subprocess.check_call') def test_install_priorities_fails(self, mock_check_call): mock_check_call.side_effect = subprocess.CalledProcessError(88, '88') + cmd = GenerateRepos(None, None) self.assertRaises(subprocess.CalledProcessError, - main._install_priorities) + cmd._install_priorities) - @mock.patch('tripleo_repos.main._get_repo') - @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') def test_install_repos_current(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current'] args.branch = 'master' args.output_path = 'test' mock_get.return_value = '[delorean]\nMr. Fusion' - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), ], @@ -169,15 +185,16 @@ class TestTripleORepos(testtools.TestCase): ], mock_write.mock_calls) - @mock.patch('tripleo_repos.main._get_repo') - @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') def test_install_repos_current_mitaka(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current'] args.branch = 'mitaka' args.output_path = 'test' mock_get.return_value = '[delorean]\nMr. Fusion' - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), ], @@ -188,28 +205,30 @@ class TestTripleORepos(testtools.TestCase): ], mock_write.mock_calls) - @mock.patch('tripleo_repos.main._get_repo') - @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') def test_install_repos_deps(self, mock_write, mock_get): args = mock.Mock() args.repos = ['deps'] args.branch = 'master' args.output_path = 'test' mock_get.return_value = '[delorean-deps]\nMr. Fusion' - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') mock_get.assert_called_once_with('roads/delorean-deps.repo', args) mock_write.assert_called_once_with('[delorean-deps]\nMr. Fusion', 'test') - @mock.patch('tripleo_repos.main._get_repo') - @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') def test_install_repos_current_tripleo(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current-tripleo'] args.branch = 'master' args.output_path = 'test' mock_get.return_value = '[delorean]\nMr. Fusion' - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current-tripleo/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), @@ -220,15 +239,16 @@ class TestTripleORepos(testtools.TestCase): ], mock_write.mock_calls) - @mock.patch('tripleo_repos.main._get_repo') - @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') def test_install_repos_current_tripleo_dev(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current-tripleo-dev'] args.branch = 'master' args.output_path = 'test' mock_get.return_value = '[delorean]\nMr. Fusion' - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') mock_get.assert_any_call('roads/delorean-deps.repo', args) # This is the wrong name for the deps repo, but I'm not bothered # enough by that to mess with mocking multiple different calls. @@ -240,18 +260,19 @@ class TestTripleORepos(testtools.TestCase): mock_get.assert_called_with('roads/current/delorean.repo', args) mock_write.assert_called_with('[delorean]\npriority=10\n%s\n' 'Mr. Fusion' % - main.INCLUDE_PKGS, 'test', + C.INCLUDE_PKGS, 'test', name='delorean') - @mock.patch('tripleo_repos.main._get_repo') - @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') def test_install_repos_tripleo_ci_testing(self, mock_write, mock_get): args = mock.Mock() args.repos = ['tripleo-ci-testing'] args.branch = 'master' args.output_path = 'test' mock_get.return_value = '[delorean]\nMr. Fusion' - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/tripleo-ci-testing/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), @@ -262,15 +283,16 @@ class TestTripleORepos(testtools.TestCase): ], mock_write.mock_calls) - @mock.patch('tripleo_repos.main._get_repo') - @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') def test_install_repos_current_tripleo_rdo(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current-tripleo-rdo'] args.branch = 'master' args.output_path = 'test' mock_get.return_value = '[delorean]\nMr. Fusion' - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current-tripleo-rdo/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), @@ -283,8 +305,8 @@ class TestTripleORepos(testtools.TestCase): @ddt.data('liberty', 'mitaka', 'newton', 'ocata', 'pike', 'queens', 'rocky', 'stein', 'master') - @mock.patch('tripleo_repos.main._write_repo') - @mock.patch('tripleo_repos.main._create_ceph') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._create_ceph') def test_install_repos_ceph(self, branch, mock_create_ceph, @@ -309,18 +331,20 @@ class TestTripleORepos(testtools.TestCase): args.output_path = 'test' mock_repo = '[centos-ceph-luminous]\nMr. Fusion' mock_create_ceph.return_value = mock_repo - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') mock_create_ceph.assert_called_once_with(args, ceph_release[branch]) mock_write_repo.assert_called_once_with(mock_repo, 'test') - @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') def test_install_repos_opstools(self, mock_write): args = mock.Mock() args.repos = ['opstools'] args.branch = 'master' args.output_path = 'test' args.mirror = 'http://foo' - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') expected_repo = ('\n[tripleo-centos-opstools]\n' 'name=tripleo-centos-opstools\n' 'baseurl=http://foo/centos/7/opstools/$basearch/\n' @@ -330,7 +354,7 @@ class TestTripleORepos(testtools.TestCase): 'test') @mock.patch('requests.get') - @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') def test_install_repos_deps_mirror(self, mock_write, mock_get): args = mock.Mock() args.repos = ['deps'] @@ -365,18 +389,20 @@ enabled=1 ''' mock_get.return_value = mock.Mock(text=fake_repo, status_code=200) - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') mock_write.assert_called_once_with(expected_repo, 'test') def test_install_repos_invalid(self): args = mock.Mock() args.repos = ['roads?'] - self.assertRaises(main.InvalidArguments, main._install_repos, args, + cmd = GenerateRepos(None, None) + self.assertRaises(E.InvalidArguments, cmd._install_repos, args, 'roads/') - @mock.patch('tripleo_repos.main._get_repo') - @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') def test_install_repos_centos8(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current'] @@ -386,7 +412,8 @@ enabled=1 args.stream = False args.mirror = 'mirror' mock_get.return_value = '[delorean]\nMr. Fusion' - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), ], @@ -409,8 +436,8 @@ enabled=1 ], mock_write.mock_calls) - @mock.patch('tripleo_repos.main._get_repo') - @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') def test_install_repos_centos8_stream(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current'] @@ -421,7 +448,8 @@ enabled=1 args.no_stream = False args.mirror = 'mirror' mock_get.return_value = '[delorean]\nMr. Fusion' - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), ], @@ -444,8 +472,8 @@ enabled=1 ], mock_write.mock_calls) - @mock.patch('tripleo_repos.main._get_repo') - @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') + @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') def test_install_repos_centos8_no_stream(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current'] @@ -456,7 +484,8 @@ enabled=1 args.no_stream = True args.mirror = 'mirror' mock_get.return_value = '[delorean]\nMr. Fusion' - main._install_repos(args, 'roads/') + cmd = GenerateRepos(None, None) + cmd._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), ], @@ -481,20 +510,25 @@ enabled=1 def test_write_repo(self): m = mock.mock_open() - with mock.patch('tripleo_repos.main.open', m, create=True): - main._write_repo('#Doc\n[delorean]\nThis=Heavy', 'test') - m.assert_called_once_with('test/delorean.repo', 'w') + with mock.patch('builtins.open', m): + cmd = GenerateRepos(None, None) + cmd._write_repo('#Doc\n[delorean]\nThis=Heavy', 'test') + + m.assert_called_with('test/delorean.repo', 'w') m().write.assert_called_once_with('#Doc\n[delorean]\nThis=Heavy') def test_write_repo_invalid(self): - self.assertRaises(main.NoRepoTitle, main._write_repo, 'Great Scot!', + self.assertRaises(E.NoRepoTitle, + GenerateRepos(None, None)._write_repo, 'Great Scot!', 'test') def test_parse_args(self): with mock.patch.object(sys, 'argv', ['', 'current', 'deps', '-d', 'centos7', '-b', 'liberty', '-o', 'test']): - args = main._parse_args('centos', '8') + + cmd = GenerateRepos(None, None) + args = cmd.get_parser('NAME').parse_args() self.assertEqual(['current', 'deps'], args.repos) self.assertEqual('centos7', args.distro) self.assertEqual('liberty', args.branch) @@ -505,35 +539,41 @@ enabled=1 'centos7', '--branch', 'mitaka', '--output-path', 'test']): - args = main._parse_args('centos', '8') + cmd = GenerateRepos(None, None) + args = cmd.get_parser('NAME').parse_args() self.assertEqual(['current'], args.repos) self.assertEqual('centos7', args.distro) self.assertEqual('mitaka', args.branch) self.assertEqual('test', args.output_path) def test_change_priority(self): - result = main._change_priority('[delorean]\npriority=1', 10) + cmd = GenerateRepos(None, None) + result = cmd._change_priority('[delorean]\npriority=1', 10) self.assertEqual('[delorean]\npriority=10', result) def test_change_priority_none(self): - result = main._change_priority('[delorean]', 10) + cmd = GenerateRepos(None, None) + result = cmd._change_priority('[delorean]', 10) self.assertEqual('[delorean]\npriority=10', result) def test_change_priority_none_muilti(self): data = "[repo1]\n[repo2]\n" expected = "[repo1]\n{0}\n[repo2]\n{0}\n".format("priority=10") - result = main._change_priority(data, 10) + cmd = GenerateRepos(None, None) + result = cmd._change_priority(data, 10) self.assertEqual(expected, result) def test_add_includepkgs(self): data = "[repo1]\n[repo2]" - expected = "[repo1]\n{0}\n[repo2]\n{0}".format(main.INCLUDE_PKGS) - result = main._add_includepkgs(data) + expected = "[repo1]\n{0}\n[repo2]\n{0}".format(C.INCLUDE_PKGS) + cmd = GenerateRepos(None, None) + result = cmd._add_includepkgs(data) self.assertEqual(expected, result) def test_create_ceph(self): mock_args = mock.Mock(mirror='http://foo') - result = main._create_ceph(mock_args, 'jewel') + cmd = GenerateRepos(None, None) + result = cmd._create_ceph(mock_args, 'jewel') expected_repo = ''' [tripleo-centos-ceph-jewel] name=tripleo-centos-ceph-jewel @@ -568,7 +608,8 @@ enabled=1 rdo_mirror='http://bar', distro='centos', old_mirror='http://mirror.centos.org') - result = main._inject_mirrors(start_repo, mock_args) + cmd = GenerateRepos(None, None) + result = cmd._inject_mirrors(start_repo, mock_args) self.assertEqual(expected, result) def test_inject_mirrors_rhel(self): @@ -596,7 +637,8 @@ enabled=1 rdo_mirror='http://bar', distro='rhel', old_mirror='https://some') - result = main._inject_mirrors(start_repo, mock_args) + cmd = GenerateRepos(None, None) + result = cmd._inject_mirrors(start_repo, mock_args) self.assertEqual(expected, result) def test_inject_mirrors_no_match(self): @@ -610,24 +652,28 @@ enabled=1 distro='centos') # If a user has a mirror whose repos already point at itself then # the _inject_mirrors call should be a noop. - self.assertEqual(start_repo, main._inject_mirrors(start_repo, - mock_args)) + cmd = GenerateRepos(None, None) + self.assertEqual(start_repo, cmd._inject_mirrors(start_repo, + mock_args)) @mock.patch('subprocess.check_call') def test_run_pkg_clean(self, mock_check_call): - main._run_pkg_clean('centos7') + cmd = GenerateRepos(None, None) + cmd._run_pkg_clean('centos7') mock_check_call.assert_called_once_with(['yum', 'clean', 'metadata']) @mock.patch('subprocess.check_call') def test_run_pkg_clean_fedora(self, mock_check_call): - main._run_pkg_clean('fedora') + cmd = GenerateRepos(None, None) + cmd._run_pkg_clean('fedora') mock_check_call.assert_called_once_with(['dnf', 'clean', 'metadata']) @mock.patch('subprocess.check_call') def test_run_pkg_clean_fails(self, mock_check_call): mock_check_call.side_effect = subprocess.CalledProcessError(88, '88') + cmd = GenerateRepos(None, None) self.assertRaises(subprocess.CalledProcessError, - main._run_pkg_clean, ['centos7']) + cmd._run_pkg_clean, ['centos7']) class TestValidate(testtools.TestCase): @@ -639,68 +685,69 @@ class TestValidate(testtools.TestCase): self.args.distro = 'centos7' self.args.stream = False self.args.no_stream = False + self.cmd = GenerateRepos(None, None) def test_good(self): - main._validate_args(self.args, '') + self.cmd._validate_args(self.args, '') def test_current_and_tripleo_dev(self): self.args.repos = ['current', 'current-tripleo-dev'] - self.assertRaises(main.InvalidArguments, main._validate_args, + self.assertRaises(E.InvalidArguments, self.cmd._validate_args, self.args, '') def test_tripleo_ci_testing_and_current_tripleo(self): self.args.repos = ['current-tripleo', 'tripleo-ci-testing'] - self.assertRaises(main.InvalidArguments, main._validate_args, + self.assertRaises(E.InvalidArguments, self.cmd._validate_args, self.args, '') def test_tripleo_ci_testing_and_ceph_opstools_allowed(self): self.args.repos = ['ceph', 'opstools', 'tripleo-ci-testing'] - main._validate_args(self.args, '') + self.cmd._validate_args(self.args, '') def test_tripleo_ci_testing_and_deps_allowed(self): self.args.repos = ['deps', 'tripleo-ci-testing'] - main._validate_args(self.args, '') + self.cmd._validate_args(self.args, '') def test_ceph_and_tripleo_dev(self): self.args.repos = ['current-tripleo-dev', 'ceph'] - self.args.output_path = main.DEFAULT_OUTPUT_PATH - main._validate_args(self.args, '') + self.args.output_path = C.DEFAULT_OUTPUT_PATH + self.cmd._validate_args(self.args, '') def test_deps_and_tripleo_dev(self): self.args.repos = ['deps', 'current-tripleo-dev'] - self.assertRaises(main.InvalidArguments, main._validate_args, + self.assertRaises(E.InvalidArguments, self.cmd._validate_args, self.args, '') def test_current_and_tripleo(self): self.args.repos = ['current', 'current-tripleo'] - self.assertRaises(main.InvalidArguments, main._validate_args, + self.assertRaises(E.InvalidArguments, self.cmd._validate_args, self.args, '') def test_deps_and_tripleo_allowed(self): self.args.repos = ['deps', 'current-tripleo'] - main._validate_args(self.args, '') + self.cmd._validate_args(self.args, '') def test_invalid_distro(self): self.args.distro = 'Jigawatts 1.21' - self.assertRaises(main.InvalidArguments, main._validate_args, + self.assertRaises(E.InvalidArguments, self.cmd._validate_args, self.args, '') def test_invalid_stream(self): self.args.stream = True - self.assertRaises(main.InvalidArguments, main._validate_args, + self.assertRaises(E.InvalidArguments, self.cmd._validate_args, self.args, 'CentOS 8') def test_invalid_no_stream(self): self.args.stream = False self.args.no_stream = True - self.assertRaises(main.InvalidArguments, main._validate_args, + self.assertRaises(E.InvalidArguments, self.cmd._validate_args, self.args, 'CentOS 8 Stream') def test_validate_distro_repos(self): - self.assertTrue(main._validate_distro_repos(self.args)) + self.assertTrue(self.cmd._validate_distro_repos(self.args)) def test_validate_distro_repos_fedora_tripleo_dev(self): self.args.distro = 'fedora' self.args.repos = ['current-tripleo-dev'] - self.assertRaises(main.InvalidArguments, main._validate_distro_repos, + self.assertRaises(E.InvalidArguments, self.cmd._validate_distro_repos, self.args)