diff --git a/requirements.txt b/requirements.txt index ce3fde9..2623409 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,5 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -cliff>=3.8.0 pbr!=2.1.0,>=2.0.0 # Apache-2.0 requests>=2.10.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index c08bb10..6392b0d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,5 +28,3 @@ packages = [entry_points] console_scripts = tripleo-repos = tripleo_repos.main:main -tripleo_repos.cm = - generate = tripleo_repos.generate_repos:GenerateRepos diff --git a/tripleo_repos/constants.py b/tripleo_repos/constants.py deleted file mode 100644 index 7f1db91..0000000 --- a/tripleo_repos/constants.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Red Hat, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - - -# Regexes -TITLE_RE = re.compile('\\[(.*)\\]') -NAME_RE = re.compile('name=(.+)') -PRIORITY_RE = re.compile('priority=\\d+') - - -# Packages to be included from delorean-current when using current-tripleo -INCLUDE_PKGS = ('includepkgs=instack,instack-undercloud,' - 'os-apply-config,os-collect-config,os-net-config,' - 'os-refresh-config,python*-tripleoclient,' - 'openstack-tripleo-*,openstack-puppet-modules,' - 'ansible-role-tripleo*,puppet-*,python*-tripleo-common,' - 'python*-paunch*,tripleo-ansible,ansible-config_template') - -# RHEL is only provided to licensed cloud providers via RHUI -DEFAULT_MIRROR_MAP = { - 'fedora': 'https://mirrors.fedoraproject.org', - 'centos': 'http://mirror.centos.org', - 'ubi': 'http://mirror.centos.org', - 'rhel': 'https://trunk.rdoproject.org', -} - -# unversioned fedora added for backwards compatibility -SUPPORTED_DISTROS = [ - ('centos', '7'), - ('centos', '8'), - ('fedora', ''), - ('rhel', '8'), - ('ubi', '8') # a subcase of the rhel distro -] - -DEFAULT_OUTPUT_PATH = '/etc/yum.repos.d' -DEFAULT_RDO_MIRROR = 'https://trunk.rdoproject.org' diff --git a/tripleo_repos/exceptions.py b/tripleo_repos/exceptions.py deleted file mode 100644 index ec7f87d..0000000 --- a/tripleo_repos/exceptions.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Red Hat, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class InvalidArguments(Exception): - pass - - -class NoRepoTitle(Exception): - pass diff --git a/tripleo_repos/generate_repos.py b/tripleo_repos/generate_repos.py deleted file mode 100644 index 56749c7..0000000 --- a/tripleo_repos/generate_repos.py +++ /dev/null @@ -1,467 +0,0 @@ -#!/usr/bin/python -# Copyright 2020 Red Hat, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import os -import re -import requests -import subprocess - -from cliff.command import Command - -import tripleo_repos.constants as C -import tripleo_repos.templates as T -import tripleo_repos.exceptions as e - - -class GenerateRepos(Command): - """Command to generate the repos""" - - log = logging.getLogger(__name__) - - def __init__(self, app, app_args, cmd_name=None): - super(GenerateRepos, self).__init__(app, app_args, cmd_name) - distro_info = self._get_distro() - self.distro_id = distro_info[0] - self.distro_major_version_id = distro_info[1] - self.distro_name = distro_info[2] - self.default_mirror = C.DEFAULT_MIRROR_MAP[self.distro_id] - - def take_action(self, parsed_args): - self.log.debug('Running GenerateRepos command') - if parsed_args.no_stream: - parsed_args.stream = False - parsed_args.old_mirror = self.default_mirror - - self._validate_args(parsed_args, self.distro_name) - base_path = self._get_base_path(parsed_args) - if parsed_args.distro in ['centos7']: - self._install_priorities() - self._remove_existing(parsed_args) - self._install_repos(parsed_args, base_path) - self._run_pkg_clean(parsed_args.distro) - - def get_parser(self, prog_name): - parser = super(GenerateRepos, self).get_parser(prog_name) - - distro = "{}{}".format(self.distro_id, self.distro_major_version_id) - - # Calculating arguments default from constants - distro_choices = ["".join(distro_pair) - for distro_pair in C.SUPPORTED_DISTROS] - - parser.add_argument('repos', metavar='REPO', nargs='+', - choices=['current', 'deps', 'current-tripleo', - 'current-tripleo-dev', 'ceph', 'opstools', - 'tripleo-ci-testing', - 'current-tripleo-rdo'], - help='A list of repos. Available repos: ' - '%(choices)s. The deps repo will always be ' - 'included when using current or ' - 'current-tripleo. current-tripleo-dev ' - 'downloads the current-tripleo, current, and ' - 'deps repos, but sets the current repo to ' - 'only be used for TripleO projects. ' - 'It also modifies each repo\'s priority so ' - 'packages are installed from the appropriate ' - 'location.') - parser.add_argument('-d', '--distro', - default=distro, - choices=distro_choices, - nargs='?', - help='Target distro with default detected at ' - 'runtime.' - ) - parser.add_argument('-b', '--branch', - default='master', - help='Target branch. Should be the lowercase ' - 'name of the OpenStack release. e.g. liberty') - parser.add_argument('-o', '--output-path', - default=C.DEFAULT_OUTPUT_PATH, - help='Directory in which to save the selected ' - 'repos.') - parser.add_argument('--mirror', - default=self.default_mirror, - help='Server from which to install base OS ' - 'packages. Default value is based on distro ' - 'param.') - parser.add_argument('--rdo-mirror', - default=C.DEFAULT_RDO_MIRROR, - help='Server from which to install RDO packages.') - - stream_group = parser.add_mutually_exclusive_group() - stream_group.add_argument('--stream', - action='store_true', - default=True, - help='Enable stream support for CentOS ' - 'repos') - stream_group.add_argument('--no-stream', - action='store_true', - default=False, - help='Disable stream support for CentOS ' - 'repos') - - return parser - - def _run_pkg_clean(self, distro): - pkg_mgr = 'yum' if distro == 'centos7' else 'dnf' - try: - subprocess.check_call([pkg_mgr, 'clean', 'metadata']) - except subprocess.CalledProcessError: - self.log.error('Failed to clean yum metadata.') - raise - - def _inject_mirrors(self, content, args): - """Replace any references to the default mirrors in repo content - - In some cases we want to use mirrors whose repo files still point to - the default servers. If the user specified to use the mirror, we want - to replace any such references with the mirror address. This function - handles that by using a regex to swap out the baseurl server. - """ - - content = re.sub('baseurl=%s' % C.DEFAULT_RDO_MIRROR, - 'baseurl=%s' % args.rdo_mirror, - content) - - if args.old_mirror: - content = re.sub('baseurl=%s' % args.old_mirror, - 'baseurl=%s' % args.mirror, - content) - - return content - - def _get_repo(self, path, args): - r = requests.get(path) - if r.status_code == 200: - return self._inject_mirrors(r.text, args) - else: - r.raise_for_status() - - def _write_repo(self, content, target, name=None): - if not name: - m = C.TITLE_RE.search(content) - if not m: - raise e.NoRepoTitle( - 'Could not find repo title in: \n%s' % content) - name = m.group(1) - # centos-8 dlrn repos have changed. repos per component - # are folded into a single repo. - if 'component' in name: - name = 'delorean' - filename = name + '.repo' - filename = os.path.join(target, filename) - with open(filename, 'w') as f: - f.write(content) - self.log.info('Installed repo %s to %s' % (name, filename)) - - def _change_priority(self, content, new_priority): - new_content = C.PRIORITY_RE.sub('priority=%d' % new_priority, content) - # This shouldn't happen, but let's be safe. - if not C.PRIORITY_RE.search(new_content): - new_content = [] - for line in content.split("\n"): - new_content.append(line) - if line.startswith('['): - new_content.append('priority=%d' % new_priority) - new_content = "\n".join(new_content) - return new_content - - def _create_ceph(self, args, release): - """Generate a Ceph repo file for release""" - centos_release = '7' if args.distro == 'centos7' else '8' - return T.CEPH_REPO_TEMPLATE % {'centos_release': centos_release, - 'ceph_release': release, - 'mirror': args.mirror} - - def _add_includepkgs(self, content): - new_content = [] - for line in content.split("\n"): - new_content.append(line) - if line.startswith('['): - new_content.append(C.INCLUDE_PKGS) - return "\n".join(new_content) - - # TODO: This need to be refactored - def _install_repos(self, args, base_path): - def install_deps(args, base_path): - content = self._get_repo(base_path + 'delorean-deps.repo', args) - self._write_repo(content, args.output_path) - for repo in args.repos: - if repo == 'current': - content = self._get_repo( - base_path + 'current/delorean.repo', args) - self._write_repo(content, args.output_path, name='delorean') - install_deps(args, base_path) - elif repo == 'deps': - install_deps(args, base_path) - elif repo == 'current-tripleo': - content = self._get_repo( - base_path + 'current-tripleo/delorean.repo', args) - self._write_repo(content, args.output_path) - install_deps(args, base_path) - elif repo == 'current-tripleo-dev': - content = self._get_repo( - base_path + 'delorean-deps.repo', args) - self._write_repo(content, args.output_path) - content = self._get_repo( - base_path + 'current-tripleo/delorean.repo', args) - content = C.TITLE_RE.sub('[\\1-current-tripleo]', content) - content = C.NAME_RE.sub('name=\\1-current-tripleo', content) - # We need to twiddle priorities since we're mixing multiple - # repos that are generated with the same priority. - content = self._change_priority(content, 20) - self._write_repo(content, args.output_path, - name='delorean-current-tripleo') - content = self._get_repo( - base_path + 'current/delorean.repo', args) - content = self._add_includepkgs(content) - content = self._change_priority(content, 10) - self._write_repo(content, args.output_path, name='delorean') - elif repo == 'tripleo-ci-testing': - content = self._get_repo( - base_path + 'tripleo-ci-testing/delorean.repo', args) - self._write_repo(content, args.output_path) - install_deps(args, base_path) - elif repo == 'current-tripleo-rdo': - content = self._get_repo( - base_path + 'current-tripleo-rdo/delorean.repo', args) - self._write_repo(content, args.output_path) - install_deps(args, base_path) - elif repo == 'ceph': - if args.branch in ['liberty', 'mitaka']: - content = self._create_ceph(args, 'hammer') - elif args.branch in ['newton', 'ocata', 'pike']: - content = self._create_ceph(args, 'jewel') - elif args.branch in ['queens', 'rocky']: - content = self._create_ceph(args, 'luminous') - elif args.branch in ['stein', 'train', 'ussuri', 'victoria']: - content = self._create_ceph(args, 'nautilus') - else: - content = self._create_ceph(args, 'pacific') - self._write_repo(content, args.output_path) - elif repo == 'opstools': - content = T.OPSTOOLS_REPO_TEMPLATE % {'mirror': args.mirror} - self._write_repo(content, args.output_path) - else: - raise e.InvalidArguments('Invalid repo "%s" specified' % repo) - - distro = args.distro - # CentOS-8 AppStream is required for UBI-8 - if distro == 'ubi8': - if not os.path.exists("/etc/distro.repos.d"): - self.log.warning('For UBI it is recommended to create ' - '/etc/distro.repos.d and rerun!') - dp_exists = False - else: - dp_exists = True - if args.output_path == C.DEFAULT_OUTPUT_PATH and dp_exists: - distro_path = "/etc/distro.repos.d" - else: - distro_path = args.output_path - # TODO: Remove it once bugs are fixed - # Add extra options to APPSTREAM_REPO_TEMPLATE because of - # rhbz/1961558 and lpbz/1929634 - extra = '' - if args.branch in ['train', 'ussuri', 'victoria']: - extra = 'exclude=edk2-ovmf-20200602gitca407c7246bf-5*' - content = T.APPSTREAM_REPO_TEMPLATE % {'mirror': args.mirror, - 'extra': extra} - self._write_repo(content, distro_path) - content = T.BASE_REPO_TEMPLATE % {'mirror': args.mirror} - self._write_repo(content, distro_path) - distro = 'centos8' # switch it to continue as centos8 distro - - # HA, Powertools are required for CentOS-8 - if distro == 'centos8': - stream = '8' - if args.stream and not args.no_stream: - stream = stream + '-stream' - content = T.HIGHAVAILABILITY_REPO_TEMPLATE % { - 'mirror': args.mirror, 'stream': stream} - self._write_repo(content, args.output_path) - content = T.POWERTOOLS_REPO_TEMPLATE % {'mirror': args.mirror, - 'stream': stream} - self._write_repo(content, args.output_path) - - def _install_priorities(self): - try: - subprocess.check_call(['yum', 'install', '-y', - 'yum-plugin-priorities']) - except subprocess.CalledProcessError as e: - self.log.error('Failed to install yum-plugin-priorities\n%s\n%s' % - (e.cmd, e.output)) - raise - - def _get_distro(self): - """Get distro info from os-release - - returns: distro_id, distro_major_version_id, distro_name - """ - - output = subprocess.Popen( - 'source /etc/os-release && echo -e -n "$ID\n$VERSION_ID\n$NAME"', - shell=True, - stdout=subprocess.PIPE, - stderr=open(os.devnull, 'w'), - executable='/bin/bash', - universal_newlines=True).communicate() - - # distro_id and distro_version_id will always be at least an - # empty string - distro_id, distro_version_id, distro_name = output[0].split('\n') - - # if distro_version_id is empty string the major version will be empty - # string too - distro_major_version_id = distro_version_id.split('.')[0] - - # check if that is UBI subcase? - if os.path.exists('/etc/yum.repos.d/ubi.repo'): - distro_id = 'ubi' - - if (distro_id, distro_major_version_id) not in C.SUPPORTED_DISTROS: - self.log.warning( - "Unsupported platform '{}{}' detected by tripleo-repos," - " centos7 will be used unless you use CLI param to change it." - "".format(distro_id, distro_major_version_id)) - distro_id = 'centos' - distro_major_version_id = '7' - - if distro_id == 'ubi': - self.log.warning( - "Centos{} Base and AppStream will be installed for " - "this UBI distro".format(distro_major_version_id)) - - return distro_id, distro_major_version_id, distro_name - - def _remove_existing(self, args): - """Remove any delorean* or opstools repos that already exist""" - if args.distro == 'ubi8': - regex = '^(BaseOS|AppStream|delorean|tripleo-centos-' \ - '(opstools|ceph|highavailability|powertools)).*.repo' - else: - regex = '^(delorean|tripleo-centos-' \ - '(opstools|ceph|highavailability|powertools)).*.repo' - pattern = re.compile(regex) - if os.path.exists("/etc/distro.repos.d"): - paths = set( - os.listdir(args.output_path) + os.listdir( - "/etc/distro.repos.d")) - else: - paths = os.listdir(args.output_path) - for f in paths: - if pattern.match(f): - filename = os.path.join(args.output_path, f) - if os.path.exists(filename): - os.remove(filename) - self.log.info('Removed old repo "%s"' % filename) - filename = os.path.join("/etc/distro.repos.d", f) - if os.path.exists(filename): - os.remove(filename) - self.log.info('Removed old repo "%s"' % filename) - - def _get_base_path(self, args): - if args.distro == 'ubi8': - # there are no base paths for UBI that work well - distro = 'centos8' - else: - distro = args.distro - - # The mirror url with /$DISTRO$VERSION path for master branch is - # deprecated. - # The default for rdo mirrors is $DISTRO$VERSION-$BRANCH - # it should work for every (distro, branch) pair that - # makes sense - # Any exception should be corrected at source, not here. - distro_branch = '%s-%s' % (distro, args.branch) - return '%s/%s/' % (args.rdo_mirror, distro_branch) - - # Validation functions - - def _validate_args(self, args, distro_name): - self._validate_current_tripleo(args.repos) - self._validate_distro_repos(args) - self._validate_tripleo_ci_testing(args.repos) - self._validate_distro_stream(args, distro_name) - - def _validate_distro_repos(self, args): - """Validate requested repos are valid for the distro""" - valid_repos = [] - if 'fedora' in args.distro: - valid_repos = ['current', 'current-tripleo', 'ceph', 'deps', - 'tripleo-ci-testing'] - elif args.distro in ['centos7', 'centos8', 'rhel8', 'ubi8']: - valid_repos = ['ceph', 'current', 'current-tripleo', - 'current-tripleo-dev', 'deps', 'tripleo-ci-testing', - 'opstools', 'current-tripleo-rdo'] - invalid_repos = [x for x in args.repos if x not in valid_repos] - if len(invalid_repos) > 0: - raise e.InvalidArguments('{} repo(s) are not valid for {}. Valid ' - 'repos are: {}'.format(invalid_repos, - args.distro, - valid_repos)) - return True - - def _validate_tripleo_ci_testing(self, repos): - """Validate tripleo-ci-testing - - With tripleo-ci-testing for repo (currently only periodic container - build) no other repos expected except optionally deps|ceph|opstools - which is enabled regardless. - """ - if 'tripleo-ci-testing' in repos and len(repos) > 1: - if 'deps' in repos or 'ceph' in repos or 'opstools' in repos: - return True - else: - raise e.InvalidArguments('Cannot use tripleo-ci-testing at the' - ' same time as other repos, except ' - 'deps|ceph|opstools.') - return True - - def _validate_distro_stream(self, args, distro_name): - """Validate stream related args vs host - - Fails if stream is to be used but the host isn't a stream OS or - vice versa - """ - is_stream = args.stream and not args.no_stream - if is_stream and 'stream' not in distro_name.lower(): - raise e.InvalidArguments('--stream provided, but OS is not the ' - 'Stream version. Please ensure the host ' - 'is Stream.') - elif not is_stream and 'stream' in distro_name.lower(): - raise e.InvalidArguments('--no-stream provided, but OS is the ' - 'Stream version. Please ensure the host ' - 'is not the Stream version.') - return True - - def _validate_current_tripleo(self, repos): - """Validate current usage - - current and current-tripleo cannot be specified with each other and - current-tripleo-dev is a mix of current, current-tripleo and deps - so they should not be specified on the command line with each other. - """ - if 'current-tripleo' in repos and 'current' in repos: - raise e.InvalidArguments( - 'Cannot use current and current-tripleo at the same time.') - if 'current-tripleo-dev' not in repos: - return True - if 'current' in repos or 'current-tripleo' in repos or 'deps' in repos: - raise e.InvalidArguments( - 'current-tripleo-dev should not be used with any other ' - 'RDO Trunk repos.') - return True diff --git a/tripleo_repos/main.py b/tripleo_repos/main.py index b9d59b4..6f299b7 100755 --- a/tripleo_repos/main.py +++ b/tripleo_repos/main.py @@ -14,41 +14,531 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function +import argparse +import os +import re +import subprocess import sys -from cliff.app import App -from cliff.commandmanager import CommandManager +import requests -class TripleoReposApp(App): +TITLE_RE = re.compile('\\[(.*)\\]') +NAME_RE = re.compile('name=(.+)') +PRIORITY_RE = re.compile('priority=\\d+') +# Packages to be included from delorean-current when using current-tripleo +INCLUDE_PKGS = ('includepkgs=instack,instack-undercloud,' + 'os-apply-config,os-collect-config,os-net-config,' + 'os-refresh-config,python*-tripleoclient,' + 'openstack-tripleo-*,openstack-puppet-modules,' + 'ansible-role-tripleo*,puppet-*,python*-tripleo-common,' + 'python*-paunch*,tripleo-ansible,ansible-config_template') +DEFAULT_OUTPUT_PATH = '/etc/yum.repos.d' +DEFAULT_RDO_MIRROR = 'https://trunk.rdoproject.org' - def __init__(self): - super(TripleoReposApp, self).__init__( - description='Tripleo repos tool', - version='2.0', - command_manager=CommandManager('tripleo_repos.cm'), - deferred_help=True) - - def initialize_app(self, argv): - self.LOG.debug('Initializing tripleo-repos tool') - - def prepare_to_run_command(self, cmd): - self.LOG.debug('prepare_to_run_command %s', cmd.__class__.__name__) - - def clean_up(self, cmd, result, err): - self.LOG.debug('clean_up %s', cmd.__class__.__name__) - if err: - self.LOG.debug('Error: %s', err) +# RHEL is only provided to licensed cloud providers via RHUI +DEFAULT_MIRROR_MAP = { + 'fedora': 'https://mirrors.fedoraproject.org', + 'centos': 'http://mirror.centos.org', + 'ubi': 'http://mirror.centos.org', + 'rhel': 'https://trunk.rdoproject.org', +} +CEPH_REPO_TEMPLATE = ''' +[tripleo-centos-ceph-%(ceph_release)s] +name=tripleo-centos-ceph-%(ceph_release)s +baseurl=%(mirror)s/centos/%(centos_release)s/storage/$basearch/ceph-%(ceph_release)s/ +gpgcheck=0 +enabled=1 +''' +OPSTOOLS_REPO_TEMPLATE = ''' +[tripleo-centos-opstools] +name=tripleo-centos-opstools +baseurl=%(mirror)s/centos/7/opstools/$basearch/ +gpgcheck=0 +enabled=1 +''' +# centos-8 only +HIGHAVAILABILITY_REPO_TEMPLATE = ''' +[tripleo-centos-highavailability] +name=tripleo-centos-highavailability +baseurl=%(mirror)s/centos/%(stream)s/HighAvailability/$basearch/os/ +gpgcheck=0 +enabled=1 +''' +# centos-8 only +POWERTOOLS_REPO_TEMPLATE = ''' +[tripleo-centos-powertools] +name=tripleo-centos-powertools +baseurl=%(mirror)s/centos/%(stream)s/PowerTools/$basearch/os/ +gpgcheck=0 +enabled=1 +''' +# ubi-8 only +APPSTREAM_REPO_TEMPLATE = ''' +[AppStream] +name=CentOS-$releasever - AppStream +baseurl=%(mirror)s/centos/$releasever/AppStream/$basearch/os/ +gpgcheck=0 +enabled=1 +%(extra)s +''' +BASE_REPO_TEMPLATE = ''' +[BaseOS] +name=CentOS-$releasever - Base +baseurl=%(mirror)s/centos/$releasever/BaseOS/$basearch/os/ +gpgcheck=0 +enabled=1 +''' -def main(argv=sys.argv[1:]): - tripleo_app = TripleoReposApp() +# unversioned fedora added for backwards compatibility +SUPPORTED_DISTROS = [ + ('centos', '7'), + ('centos', '8'), + ('fedora', ''), + ('rhel', '8'), + ('ubi', '8') # a subcase of the rhel distro +] - # Hack to keep compatibility for now - if 'generate' not in argv: - argv.insert(0, 'generate') - return tripleo_app.run(argv) + +class InvalidArguments(Exception): + pass + + +class NoRepoTitle(Exception): + pass + + +def _get_distro(): + """Get distro info from os-release + + returns: distro_id, distro_major_version_id, distro_name + """ + output = subprocess.Popen( + 'source /etc/os-release && echo -e -n "$ID\n$VERSION_ID\n$NAME"', + shell=True, + stdout=subprocess.PIPE, + stderr=open(os.devnull, 'w'), + executable='/bin/bash', + universal_newlines=True).communicate() + + # distro_id and distro_version_id will always be at least an empty string + distro_id, distro_version_id, distro_name = output[0].split('\n') + + # if distro_version_id is empty string the major version will be empty + # string too + distro_major_version_id = distro_version_id.split('.')[0] + + # check if that is UBI subcase? + if os.path.exists('/etc/yum.repos.d/ubi.repo'): + distro_id = 'ubi' + + if (distro_id, distro_major_version_id) not in SUPPORTED_DISTROS: + print( + "WARNING: Unsupported platform '{}{}' detected by tripleo-repos," + " centos7 will be used unless you use CLI param to change it." + "".format(distro_id, distro_major_version_id), file=sys.stderr) + distro_id = 'centos' + distro_major_version_id = '7' + + if distro_id == 'ubi': + print( + "WARNING: Centos{} Base and AppStream will be installed for " + "this UBI distro".format(distro_major_version_id)) + + return distro_id, distro_major_version_id, distro_name + + +def _parse_args(distro_id, distro_major_version_id): + + distro = "{}{}".format(distro_id, distro_major_version_id) + + # Calculating arguments default from constants + default_mirror = DEFAULT_MIRROR_MAP[distro_id] + distro_choices = ["".join(distro_pair) + for distro_pair in SUPPORTED_DISTROS] + + parser = argparse.ArgumentParser( + description='Download and install repos necessary for TripleO. Note ' + 'that some of these repos require yum-plugin-priorities, ' + 'so that will also be installed.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('repos', metavar='REPO', nargs='+', + choices=['current', 'deps', 'current-tripleo', + 'current-tripleo-dev', 'ceph', 'opstools', + 'tripleo-ci-testing', 'current-tripleo-rdo'], + help='A list of repos. Available repos: ' + '%(choices)s. The deps repo will always be ' + 'included when using current or ' + 'current-tripleo. current-tripleo-dev ' + 'downloads the current-tripleo, current, and ' + 'deps repos, but sets the current repo to only ' + 'be used for TripleO projects. It also modifies ' + 'each repo\'s priority so packages are installed ' + 'from the appropriate location.') + parser.add_argument('-d', '--distro', + default=distro, + choices=distro_choices, + nargs='?', + help='Target distro with default detected at runtime. ' + ) + parser.add_argument('-b', '--branch', + default='master', + help='Target branch. Should be the lowercase name of ' + 'the OpenStack release. e.g. liberty') + parser.add_argument('-o', '--output-path', + default=DEFAULT_OUTPUT_PATH, + help='Directory in which to save the selected repos.') + parser.add_argument('--mirror', + default=default_mirror, + help='Server from which to install base OS packages. ' + 'Default value is based on distro param.') + parser.add_argument('--rdo-mirror', + default=DEFAULT_RDO_MIRROR, + help='Server from which to install RDO packages.') + stream_group = parser.add_mutually_exclusive_group() + stream_group.add_argument('--stream', + action='store_true', + default=True, + help='Enable stream support for CentOS repos') + stream_group.add_argument('--no-stream', + action='store_true', + default=False, + help='Disable stream support for CentOS repos') + + args = parser.parse_args() + if args.no_stream: + args.stream = False + args.old_mirror = default_mirror + + return args + + +def _get_repo(path, args): + r = requests.get(path) + if r.status_code == 200: + return _inject_mirrors(r.text, args) + else: + r.raise_for_status() + + +def _write_repo(content, target, name=None): + if not name: + m = TITLE_RE.search(content) + if not m: + raise NoRepoTitle('Could not find repo title in: \n%s' % content) + name = m.group(1) + # centos-8 dlrn repos have changed. repos per component + # are folded into a single repo. + if 'component' in name: + name = 'delorean' + filename = name + '.repo' + filename = os.path.join(target, filename) + with open(filename, 'w') as f: + f.write(content) + print('Installed repo %s to %s' % (name, filename)) + + +def _validate_distro_repos(args): + """Validate requested repos are valid for the distro""" + valid_repos = [] + if 'fedora' in args.distro: + valid_repos = ['current', 'current-tripleo', 'ceph', 'deps', + 'tripleo-ci-testing'] + elif args.distro in ['centos7', 'centos8', 'rhel8', 'ubi8']: + valid_repos = ['ceph', 'current', 'current-tripleo', + 'current-tripleo-dev', 'deps', 'tripleo-ci-testing', + 'opstools', 'current-tripleo-rdo'] + invalid_repos = [x for x in args.repos if x not in valid_repos] + if len(invalid_repos) > 0: + raise InvalidArguments('{} repo(s) are not valid for {}. Valid repos ' + 'are: {}'.format(invalid_repos, args.distro, + valid_repos)) + return True + + +def _validate_current_tripleo(repos): + """Validate current usage + + current and current-tripleo cannot be specified with each other and + current-tripleo-dev is a mix of current, current-tripleo and deps + so they should not be specified on the command line with each other. + """ + if 'current-tripleo' in repos and 'current' in repos: + raise InvalidArguments('Cannot use current and current-tripleo at the ' + 'same time.') + if 'current-tripleo-dev' not in repos: + return True + if 'current' in repos or 'current-tripleo' in repos or 'deps' in repos: + raise InvalidArguments('current-tripleo-dev should not be used with ' + 'any other RDO Trunk repos.') + return True + + +def _validate_tripleo_ci_testing(repos): + """Validate tripleo-ci-testing + + With tripleo-ci-testing for repo (currently only periodic container build) + no other repos expected except optionally deps|ceph|opstools + which is enabled regardless. + """ + if 'tripleo-ci-testing' in repos and len(repos) > 1: + if 'deps' in repos or 'ceph' in repos or 'opstools' in repos: + return True + else: + raise InvalidArguments('Cannot use tripleo-ci-testing at the ' + 'same time as other repos, except ' + 'deps|ceph|opstools.') + return True + + +def _validate_distro_stream(args, distro_name): + """Validate stream related args vs host + + Fails if stream is to be used but the host isn't a stream OS or vice versa + """ + is_stream = args.stream and not args.no_stream + if is_stream and 'stream' not in distro_name.lower(): + raise InvalidArguments('--stream provided, but OS is not the Stream ' + 'version. Please ensure the host is Stream.') + elif not is_stream and 'stream' in distro_name.lower(): + raise InvalidArguments('--no-stream provided, but OS is the Stream ' + 'version. Please ensure the host is not the ' + 'Stream version.') + return True + + +def _validate_args(args, distro_name): + _validate_current_tripleo(args.repos) + _validate_distro_repos(args) + _validate_tripleo_ci_testing(args.repos) + _validate_distro_stream(args, distro_name) + + +def _remove_existing(args): + """Remove any delorean* or opstools repos that already exist""" + if args.distro == 'ubi8': + regex = '^(BaseOS|AppStream|delorean|tripleo-centos-' \ + '(opstools|ceph|highavailability|powertools)).*.repo' + else: + regex = '^(delorean|tripleo-centos-' \ + '(opstools|ceph|highavailability|powertools)).*.repo' + pattern = re.compile(regex) + if os.path.exists("/etc/distro.repos.d"): + paths = set( + os.listdir(args.output_path) + os.listdir("/etc/distro.repos.d")) + else: + paths = os.listdir(args.output_path) + for f in paths: + if pattern.match(f): + filename = os.path.join(args.output_path, f) + if os.path.exists(filename): + os.remove(filename) + print('Removed old repo "%s"' % filename) + filename = os.path.join("/etc/distro.repos.d", f) + if os.path.exists(filename): + os.remove(filename) + print('Removed old repo "%s"' % filename) + + +def _get_base_path(args): + if args.distro == 'ubi8': + distro = 'centos8' # there are no base paths for UBI that work well + else: + distro = args.distro + + # The mirror url with /$DISTRO$VERSION path for master branch is + # deprecated. + # The default for rdo mirrors is $DISTRO$VERSION-$BRANCH + # it should work for every (distro, branch) pair that + # makes sense + # Any exception should be corrected at source, not here. + distro_branch = '%s-%s' % (distro, args.branch) + return '%s/%s/' % (args.rdo_mirror, distro_branch) + + +def _install_priorities(): + try: + subprocess.check_call(['yum', 'install', '-y', + 'yum-plugin-priorities']) + except subprocess.CalledProcessError as e: + print('ERROR: Failed to install yum-plugin-priorities\n%s\n%s' % + (e.cmd, e.output)) + raise + + +def _create_ceph(args, release): + """Generate a Ceph repo file for release""" + centos_release = '7' if args.distro == 'centos7' else '8' + return CEPH_REPO_TEMPLATE % {'centos_release': centos_release, + 'ceph_release': release, + 'mirror': args.mirror} + + +def _change_priority(content, new_priority): + new_content = PRIORITY_RE.sub('priority=%d' % new_priority, content) + # This shouldn't happen, but let's be safe. + if not PRIORITY_RE.search(new_content): + new_content = [] + for line in content.split("\n"): + new_content.append(line) + if line.startswith('['): + new_content.append('priority=%d' % new_priority) + new_content = "\n".join(new_content) + return new_content + + +def _add_includepkgs(content): + new_content = [] + for line in content.split("\n"): + new_content.append(line) + if line.startswith('['): + new_content.append(INCLUDE_PKGS) + return "\n".join(new_content) + + +def _inject_mirrors(content, args): + """Replace any references to the default mirrors in repo content + + In some cases we want to use mirrors whose repo files still point to the + default servers. If the user specified to use the mirror, we want to + replace any such references with the mirror address. This function + handles that by using a regex to swap out the baseurl server. + """ + + content = re.sub('baseurl=%s' % DEFAULT_RDO_MIRROR, + 'baseurl=%s' % args.rdo_mirror, + content) + + if args.old_mirror: + content = re.sub('baseurl=%s' % args.old_mirror, + 'baseurl=%s' % args.mirror, + content) + + return content + + +def _install_repos(args, base_path): + def install_deps(args, base_path): + content = _get_repo(base_path + 'delorean-deps.repo', args) + _write_repo(content, args.output_path) + + for repo in args.repos: + if repo == 'current': + content = _get_repo(base_path + 'current/delorean.repo', args) + _write_repo(content, args.output_path, name='delorean') + install_deps(args, base_path) + elif repo == 'deps': + install_deps(args, base_path) + elif repo == 'current-tripleo': + content = _get_repo(base_path + 'current-tripleo/delorean.repo', + args) + _write_repo(content, args.output_path) + install_deps(args, base_path) + elif repo == 'current-tripleo-dev': + content = _get_repo(base_path + 'delorean-deps.repo', args) + _write_repo(content, args.output_path) + content = _get_repo(base_path + 'current-tripleo/delorean.repo', + args) + content = TITLE_RE.sub('[\\1-current-tripleo]', content) + content = NAME_RE.sub('name=\\1-current-tripleo', content) + # We need to twiddle priorities since we're mixing multiple repos + # that are generated with the same priority. + content = _change_priority(content, 20) + _write_repo(content, args.output_path, + name='delorean-current-tripleo') + content = _get_repo(base_path + 'current/delorean.repo', args) + content = _add_includepkgs(content) + content = _change_priority(content, 10) + _write_repo(content, args.output_path, name='delorean') + elif repo == 'tripleo-ci-testing': + content = _get_repo(base_path + 'tripleo-ci-testing/delorean.repo', + args) + _write_repo(content, args.output_path) + install_deps(args, base_path) + elif repo == 'current-tripleo-rdo': + content = _get_repo( + base_path + 'current-tripleo-rdo/delorean.repo', args) + _write_repo(content, args.output_path) + install_deps(args, base_path) + elif repo == 'ceph': + if args.branch in ['liberty', 'mitaka']: + content = _create_ceph(args, 'hammer') + elif args.branch in ['newton', 'ocata', 'pike']: + content = _create_ceph(args, 'jewel') + elif args.branch in ['queens', 'rocky']: + content = _create_ceph(args, 'luminous') + elif args.branch in ['stein', 'train', 'ussuri', 'victoria']: + content = _create_ceph(args, 'nautilus') + else: + content = _create_ceph(args, 'pacific') + _write_repo(content, args.output_path) + elif repo == 'opstools': + content = OPSTOOLS_REPO_TEMPLATE % {'mirror': args.mirror} + _write_repo(content, args.output_path) + else: + raise InvalidArguments('Invalid repo "%s" specified' % repo) + + distro = args.distro + # CentOS-8 AppStream is required for UBI-8 + if distro == 'ubi8': + if not os.path.exists("/etc/distro.repos.d"): + print('WARNING: For UBI it is recommended to create ' + '/etc/distro.repos.d and rerun!') + dp_exists = False + else: + dp_exists = True + if args.output_path == DEFAULT_OUTPUT_PATH and dp_exists: + distro_path = "/etc/distro.repos.d" + else: + distro_path = args.output_path + # TODO: Remove it once bugs are fixed + # Add extra options to APPSTREAM_REPO_TEMPLATE because of + # rhbz/1961558 and lpbz/1929634 + extra = '' + if args.branch in ['train', 'ussuri', 'victoria']: + extra = 'exclude=edk2-ovmf-20200602gitca407c7246bf-5*' + content = APPSTREAM_REPO_TEMPLATE % {'mirror': args.mirror, + 'extra': extra} + _write_repo(content, distro_path) + content = BASE_REPO_TEMPLATE % {'mirror': args.mirror} + _write_repo(content, distro_path) + distro = 'centos8' # switch it to continue as centos8 distro + + # HA, Powertools are required for CentOS-8 + if distro == 'centos8': + stream = '8' + if args.stream and not args.no_stream: + stream = stream + '-stream' + content = HIGHAVAILABILITY_REPO_TEMPLATE % {'mirror': args.mirror, + 'stream': stream} + _write_repo(content, args.output_path) + content = POWERTOOLS_REPO_TEMPLATE % {'mirror': args.mirror, + 'stream': stream} + _write_repo(content, args.output_path) + + +def _run_pkg_clean(distro): + pkg_mgr = 'yum' if distro == 'centos7' else 'dnf' + try: + subprocess.check_call([pkg_mgr, 'clean', 'metadata']) + except subprocess.CalledProcessError: + print('ERROR: Failed to clean yum metadata.') + raise + + +def main(): + distro_id, distro_major_version_id, distro_name = _get_distro() + args = _parse_args(distro_id, distro_major_version_id) + _validate_args(args, distro_name) + base_path = _get_base_path(args) + if args.distro in ['centos7']: + _install_priorities() + _remove_existing(args) + _install_repos(args, base_path) + _run_pkg_clean(args.distro) if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + main() diff --git a/tripleo_repos/templates.py b/tripleo_repos/templates.py deleted file mode 100644 index 1952a52..0000000 --- a/tripleo_repos/templates.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Red Hat, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -CEPH_REPO_TEMPLATE = ''' -[tripleo-centos-ceph-%(ceph_release)s] -name=tripleo-centos-ceph-%(ceph_release)s -baseurl=%(mirror)s/centos/%(centos_release)s/storage/$basearch/ceph-%(ceph_release)s/ -gpgcheck=0 -enabled=1 -''' - -OPSTOOLS_REPO_TEMPLATE = ''' -[tripleo-centos-opstools] -name=tripleo-centos-opstools -baseurl=%(mirror)s/centos/7/opstools/$basearch/ -gpgcheck=0 -enabled=1 -''' - -# centos-8 only -HIGHAVAILABILITY_REPO_TEMPLATE = ''' -[tripleo-centos-highavailability] -name=tripleo-centos-highavailability -baseurl=%(mirror)s/centos/%(stream)s/HighAvailability/$basearch/os/ -gpgcheck=0 -enabled=1 -''' - -# centos-8 only -POWERTOOLS_REPO_TEMPLATE = ''' -[tripleo-centos-powertools] -name=tripleo-centos-powertools -baseurl=%(mirror)s/centos/%(stream)s/PowerTools/$basearch/os/ -gpgcheck=0 -enabled=1 -''' - -# ubi-8 only -APPSTREAM_REPO_TEMPLATE = ''' -[AppStream] -name=CentOS-$releasever - AppStream -baseurl=%(mirror)s/centos/$releasever/AppStream/$basearch/os/ -gpgcheck=0 -enabled=1 -%(extra)s -''' - -BASE_REPO_TEMPLATE = ''' -[BaseOS] -name=CentOS-$releasever - Base -baseurl=%(mirror)s/centos/$releasever/BaseOS/$basearch/os/ -gpgcheck=0 -enabled=1 -''' diff --git a/tripleo_repos/tests/test_generate_repos.py b/tripleo_repos/tests/test_main.py similarity index 76% rename from tripleo_repos/tests/test_generate_repos.py rename to tripleo_repos/tests/test_main.py index b893165..9c766bb 100644 --- a/tripleo_repos/tests/test_generate_repos.py +++ b/tripleo_repos/tests/test_main.py @@ -19,32 +19,26 @@ from unittest import mock import ddt import testtools -from tripleo_repos.generate_repos import GenerateRepos - -import tripleo_repos.exceptions as E -import tripleo_repos.constants as C +from tripleo_repos import main @ddt.ddt class TestTripleORepos(testtools.TestCase): - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_distro') + @mock.patch('tripleo_repos.main._get_distro') @mock.patch('sys.argv', ['tripleo-repos', 'current', '-d', 'centos7']) - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._run_pkg_clean') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._validate_args') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_base_path') - @mock.patch( - 'tripleo_repos.generate_repos.GenerateRepos._install_priorities') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._remove_existing') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._install_repos') + @mock.patch('tripleo_repos.main._run_pkg_clean') + @mock.patch('tripleo_repos.main._validate_args') + @mock.patch('tripleo_repos.main._get_base_path') + @mock.patch('tripleo_repos.main._install_priorities') + @mock.patch('tripleo_repos.main._remove_existing') + @mock.patch('tripleo_repos.main._install_repos') def test_main(self, mock_install, mock_remove, mock_ip, mock_gbp, mock_validate, mock_clean, mock_distro): mock_distro.return_value = ('centos', '8', 'CentOS 8') - - self.cmd = GenerateRepos(None, None) - args = self.cmd.get_parser('NAME').parse_args() + args = main._parse_args('centos', '8') mock_path = mock.Mock() mock_gbp.return_value = mock_path - self.cmd.run(args) + main.main() mock_validate.assert_called_once_with(args, 'CentOS 8') mock_gbp.assert_called_once_with(args) mock_ip.assert_called_once_with() @@ -52,23 +46,21 @@ class TestTripleORepos(testtools.TestCase): mock_install.assert_called_once_with(args, mock_path) mock_clean.assert_called_once_with('centos7') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_distro') + @mock.patch('tripleo_repos.main._get_distro') @mock.patch('sys.argv', ['tripleo-repos', 'current', '-d', 'fedora']) - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._run_pkg_clean') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._validate_args') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_base_path') - @mock.patch( - 'tripleo_repos.generate_repos.GenerateRepos._install_priorities') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._remove_existing') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._install_repos') + @mock.patch('tripleo_repos.main._run_pkg_clean') + @mock.patch('tripleo_repos.main._validate_args') + @mock.patch('tripleo_repos.main._get_base_path') + @mock.patch('tripleo_repos.main._install_priorities') + @mock.patch('tripleo_repos.main._remove_existing') + @mock.patch('tripleo_repos.main._install_repos') def test_main_fedora(self, mock_install, mock_remove, mock_ip, mock_gbp, mock_validate, mock_clean, mock_distro): mock_distro.return_value = ('centos', '8', 'CentOS 8') - self.cmd = GenerateRepos(None, None) - args = self.cmd.get_parser('NAME').parse_args() + args = main._parse_args('centos', '8') mock_path = mock.Mock() mock_gbp.return_value = mock_path - self.cmd.run(args) + main.main() mock_validate.assert_called_once_with(args, 'CentOS 8') mock_gbp.assert_called_once_with(args) assert not mock_ip.called, '_install_priorities should no tbe called' @@ -85,8 +77,7 @@ class TestTripleORepos(testtools.TestCase): fake_addr = 'http://lone/pine/mall' args = mock.Mock() args.distro = 'centos' - cmd = GenerateRepos(None, None) - content = cmd._get_repo(fake_addr, args) + content = main._get_repo(fake_addr, args) self.assertEqual('88MPH', content) mock_get.assert_called_once_with(fake_addr) @@ -96,8 +87,7 @@ class TestTripleORepos(testtools.TestCase): mock_response.status_code = 404 mock_get.return_value = mock_response fake_addr = 'http://twin/pines/mall' - cmd = GenerateRepos(None, None) - cmd._get_repo(fake_addr, mock.Mock()) + main._get_repo(fake_addr, mock.Mock()) mock_get.assert_called_once_with(fake_addr) mock_response.raise_for_status.assert_called_once_with() @@ -114,8 +104,7 @@ class TestTripleORepos(testtools.TestCase): mock_listdir.return_value = fake_list mock_args = mock.Mock() mock_args.output_path = '/etc/yum.repos.d' - cmd = GenerateRepos(None, None) - cmd._remove_existing(mock_args) + main._remove_existing(mock_args) self.assertIn(mock.call('/etc/yum.repos.d/delorean.repo'), mock_remove.mock_calls) self.assertIn(mock.call('/etc/yum.repos.d/' @@ -138,8 +127,7 @@ class TestTripleORepos(testtools.TestCase): args.branch = 'master' args.distro = 'centos7' args.rdo_mirror = 'http://trunk.rdoproject.org' - cmd = GenerateRepos(None, None) - path = cmd._get_base_path(args) + path = main._get_base_path(args) self.assertEqual('http://trunk.rdoproject.org/centos7-master/', path) def test_get_base_path_fedora(self): @@ -147,34 +135,30 @@ class TestTripleORepos(testtools.TestCase): args.branch = 'master' args.distro = 'fedora' args.rdo_mirror = 'http://trunk.rdoproject.org' - cmd = GenerateRepos(None, None) - path = cmd._get_base_path(args) + path = main._get_base_path(args) self.assertEqual('http://trunk.rdoproject.org/fedora-master/', path) @mock.patch('subprocess.check_call') def test_install_priorities(self, mock_check_call): - cmd = GenerateRepos(None, None) - cmd._install_priorities() + main._install_priorities() mock_check_call.assert_called_once_with(['yum', 'install', '-y', 'yum-plugin-priorities']) @mock.patch('subprocess.check_call') def test_install_priorities_fails(self, mock_check_call): mock_check_call.side_effect = subprocess.CalledProcessError(88, '88') - cmd = GenerateRepos(None, None) self.assertRaises(subprocess.CalledProcessError, - cmd._install_priorities) + main._install_priorities) - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.main._get_repo') + @mock.patch('tripleo_repos.main._write_repo') def test_install_repos_current(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current'] args.branch = 'master' args.output_path = 'test' mock_get.return_value = '[delorean]\nMr. Fusion' - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), ], @@ -185,16 +169,15 @@ class TestTripleORepos(testtools.TestCase): ], mock_write.mock_calls) - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.main._get_repo') + @mock.patch('tripleo_repos.main._write_repo') def test_install_repos_current_mitaka(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current'] args.branch = 'mitaka' args.output_path = 'test' mock_get.return_value = '[delorean]\nMr. Fusion' - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), ], @@ -205,30 +188,28 @@ class TestTripleORepos(testtools.TestCase): ], mock_write.mock_calls) - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.main._get_repo') + @mock.patch('tripleo_repos.main._write_repo') def test_install_repos_deps(self, mock_write, mock_get): args = mock.Mock() args.repos = ['deps'] args.branch = 'master' args.output_path = 'test' mock_get.return_value = '[delorean-deps]\nMr. Fusion' - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') mock_get.assert_called_once_with('roads/delorean-deps.repo', args) mock_write.assert_called_once_with('[delorean-deps]\nMr. Fusion', 'test') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.main._get_repo') + @mock.patch('tripleo_repos.main._write_repo') def test_install_repos_current_tripleo(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current-tripleo'] args.branch = 'master' args.output_path = 'test' mock_get.return_value = '[delorean]\nMr. Fusion' - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current-tripleo/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), @@ -239,16 +220,15 @@ class TestTripleORepos(testtools.TestCase): ], mock_write.mock_calls) - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.main._get_repo') + @mock.patch('tripleo_repos.main._write_repo') def test_install_repos_current_tripleo_dev(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current-tripleo-dev'] args.branch = 'master' args.output_path = 'test' mock_get.return_value = '[delorean]\nMr. Fusion' - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') mock_get.assert_any_call('roads/delorean-deps.repo', args) # This is the wrong name for the deps repo, but I'm not bothered # enough by that to mess with mocking multiple different calls. @@ -260,19 +240,18 @@ class TestTripleORepos(testtools.TestCase): mock_get.assert_called_with('roads/current/delorean.repo', args) mock_write.assert_called_with('[delorean]\npriority=10\n%s\n' 'Mr. Fusion' % - C.INCLUDE_PKGS, 'test', + main.INCLUDE_PKGS, 'test', name='delorean') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.main._get_repo') + @mock.patch('tripleo_repos.main._write_repo') def test_install_repos_tripleo_ci_testing(self, mock_write, mock_get): args = mock.Mock() args.repos = ['tripleo-ci-testing'] args.branch = 'master' args.output_path = 'test' mock_get.return_value = '[delorean]\nMr. Fusion' - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/tripleo-ci-testing/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), @@ -283,16 +262,15 @@ class TestTripleORepos(testtools.TestCase): ], mock_write.mock_calls) - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.main._get_repo') + @mock.patch('tripleo_repos.main._write_repo') def test_install_repos_current_tripleo_rdo(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current-tripleo-rdo'] args.branch = 'master' args.output_path = 'test' mock_get.return_value = '[delorean]\nMr. Fusion' - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current-tripleo-rdo/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), @@ -305,8 +283,8 @@ class TestTripleORepos(testtools.TestCase): @ddt.data('liberty', 'mitaka', 'newton', 'ocata', 'pike', 'queens', 'rocky', 'stein', 'master') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._create_ceph') + @mock.patch('tripleo_repos.main._write_repo') + @mock.patch('tripleo_repos.main._create_ceph') def test_install_repos_ceph(self, branch, mock_create_ceph, @@ -331,20 +309,18 @@ class TestTripleORepos(testtools.TestCase): args.output_path = 'test' mock_repo = '[centos-ceph-luminous]\nMr. Fusion' mock_create_ceph.return_value = mock_repo - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') mock_create_ceph.assert_called_once_with(args, ceph_release[branch]) mock_write_repo.assert_called_once_with(mock_repo, 'test') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.main._write_repo') def test_install_repos_opstools(self, mock_write): args = mock.Mock() args.repos = ['opstools'] args.branch = 'master' args.output_path = 'test' args.mirror = 'http://foo' - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') expected_repo = ('\n[tripleo-centos-opstools]\n' 'name=tripleo-centos-opstools\n' 'baseurl=http://foo/centos/7/opstools/$basearch/\n' @@ -354,7 +330,7 @@ class TestTripleORepos(testtools.TestCase): 'test') @mock.patch('requests.get') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.main._write_repo') def test_install_repos_deps_mirror(self, mock_write, mock_get): args = mock.Mock() args.repos = ['deps'] @@ -389,20 +365,18 @@ enabled=1 ''' mock_get.return_value = mock.Mock(text=fake_repo, status_code=200) - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') mock_write.assert_called_once_with(expected_repo, 'test') def test_install_repos_invalid(self): args = mock.Mock() args.repos = ['roads?'] - cmd = GenerateRepos(None, None) - self.assertRaises(E.InvalidArguments, cmd._install_repos, args, + self.assertRaises(main.InvalidArguments, main._install_repos, args, 'roads/') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.main._get_repo') + @mock.patch('tripleo_repos.main._write_repo') def test_install_repos_centos8(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current'] @@ -412,8 +386,7 @@ enabled=1 args.stream = False args.mirror = 'mirror' mock_get.return_value = '[delorean]\nMr. Fusion' - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), ], @@ -436,8 +409,8 @@ enabled=1 ], mock_write.mock_calls) - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.main._get_repo') + @mock.patch('tripleo_repos.main._write_repo') def test_install_repos_centos8_stream(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current'] @@ -448,8 +421,7 @@ enabled=1 args.no_stream = False args.mirror = 'mirror' mock_get.return_value = '[delorean]\nMr. Fusion' - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), ], @@ -472,8 +444,8 @@ enabled=1 ], mock_write.mock_calls) - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._get_repo') - @mock.patch('tripleo_repos.generate_repos.GenerateRepos._write_repo') + @mock.patch('tripleo_repos.main._get_repo') + @mock.patch('tripleo_repos.main._write_repo') def test_install_repos_centos8_no_stream(self, mock_write, mock_get): args = mock.Mock() args.repos = ['current'] @@ -484,8 +456,7 @@ enabled=1 args.no_stream = True args.mirror = 'mirror' mock_get.return_value = '[delorean]\nMr. Fusion' - cmd = GenerateRepos(None, None) - cmd._install_repos(args, 'roads/') + main._install_repos(args, 'roads/') self.assertEqual([mock.call('roads/current/delorean.repo', args), mock.call('roads/delorean-deps.repo', args), ], @@ -510,25 +481,20 @@ enabled=1 def test_write_repo(self): m = mock.mock_open() - with mock.patch('builtins.open', m): - cmd = GenerateRepos(None, None) - cmd._write_repo('#Doc\n[delorean]\nThis=Heavy', 'test') - - m.assert_called_with('test/delorean.repo', 'w') + with mock.patch('tripleo_repos.main.open', m, create=True): + main._write_repo('#Doc\n[delorean]\nThis=Heavy', 'test') + m.assert_called_once_with('test/delorean.repo', 'w') m().write.assert_called_once_with('#Doc\n[delorean]\nThis=Heavy') def test_write_repo_invalid(self): - self.assertRaises(E.NoRepoTitle, - GenerateRepos(None, None)._write_repo, 'Great Scot!', + self.assertRaises(main.NoRepoTitle, main._write_repo, 'Great Scot!', 'test') def test_parse_args(self): with mock.patch.object(sys, 'argv', ['', 'current', 'deps', '-d', 'centos7', '-b', 'liberty', '-o', 'test']): - - cmd = GenerateRepos(None, None) - args = cmd.get_parser('NAME').parse_args() + args = main._parse_args('centos', '8') self.assertEqual(['current', 'deps'], args.repos) self.assertEqual('centos7', args.distro) self.assertEqual('liberty', args.branch) @@ -539,41 +505,35 @@ enabled=1 'centos7', '--branch', 'mitaka', '--output-path', 'test']): - cmd = GenerateRepos(None, None) - args = cmd.get_parser('NAME').parse_args() + args = main._parse_args('centos', '8') self.assertEqual(['current'], args.repos) self.assertEqual('centos7', args.distro) self.assertEqual('mitaka', args.branch) self.assertEqual('test', args.output_path) def test_change_priority(self): - cmd = GenerateRepos(None, None) - result = cmd._change_priority('[delorean]\npriority=1', 10) + result = main._change_priority('[delorean]\npriority=1', 10) self.assertEqual('[delorean]\npriority=10', result) def test_change_priority_none(self): - cmd = GenerateRepos(None, None) - result = cmd._change_priority('[delorean]', 10) + result = main._change_priority('[delorean]', 10) self.assertEqual('[delorean]\npriority=10', result) def test_change_priority_none_muilti(self): data = "[repo1]\n[repo2]\n" expected = "[repo1]\n{0}\n[repo2]\n{0}\n".format("priority=10") - cmd = GenerateRepos(None, None) - result = cmd._change_priority(data, 10) + result = main._change_priority(data, 10) self.assertEqual(expected, result) def test_add_includepkgs(self): data = "[repo1]\n[repo2]" - expected = "[repo1]\n{0}\n[repo2]\n{0}".format(C.INCLUDE_PKGS) - cmd = GenerateRepos(None, None) - result = cmd._add_includepkgs(data) + expected = "[repo1]\n{0}\n[repo2]\n{0}".format(main.INCLUDE_PKGS) + result = main._add_includepkgs(data) self.assertEqual(expected, result) def test_create_ceph(self): mock_args = mock.Mock(mirror='http://foo') - cmd = GenerateRepos(None, None) - result = cmd._create_ceph(mock_args, 'jewel') + result = main._create_ceph(mock_args, 'jewel') expected_repo = ''' [tripleo-centos-ceph-jewel] name=tripleo-centos-ceph-jewel @@ -608,8 +568,7 @@ enabled=1 rdo_mirror='http://bar', distro='centos', old_mirror='http://mirror.centos.org') - cmd = GenerateRepos(None, None) - result = cmd._inject_mirrors(start_repo, mock_args) + result = main._inject_mirrors(start_repo, mock_args) self.assertEqual(expected, result) def test_inject_mirrors_rhel(self): @@ -637,8 +596,7 @@ enabled=1 rdo_mirror='http://bar', distro='rhel', old_mirror='https://some') - cmd = GenerateRepos(None, None) - result = cmd._inject_mirrors(start_repo, mock_args) + result = main._inject_mirrors(start_repo, mock_args) self.assertEqual(expected, result) def test_inject_mirrors_no_match(self): @@ -652,28 +610,24 @@ enabled=1 distro='centos') # If a user has a mirror whose repos already point at itself then # the _inject_mirrors call should be a noop. - cmd = GenerateRepos(None, None) - self.assertEqual(start_repo, cmd._inject_mirrors(start_repo, - mock_args)) + self.assertEqual(start_repo, main._inject_mirrors(start_repo, + mock_args)) @mock.patch('subprocess.check_call') def test_run_pkg_clean(self, mock_check_call): - cmd = GenerateRepos(None, None) - cmd._run_pkg_clean('centos7') + main._run_pkg_clean('centos7') mock_check_call.assert_called_once_with(['yum', 'clean', 'metadata']) @mock.patch('subprocess.check_call') def test_run_pkg_clean_fedora(self, mock_check_call): - cmd = GenerateRepos(None, None) - cmd._run_pkg_clean('fedora') + main._run_pkg_clean('fedora') mock_check_call.assert_called_once_with(['dnf', 'clean', 'metadata']) @mock.patch('subprocess.check_call') def test_run_pkg_clean_fails(self, mock_check_call): mock_check_call.side_effect = subprocess.CalledProcessError(88, '88') - cmd = GenerateRepos(None, None) self.assertRaises(subprocess.CalledProcessError, - cmd._run_pkg_clean, ['centos7']) + main._run_pkg_clean, ['centos7']) class TestValidate(testtools.TestCase): @@ -685,69 +639,68 @@ class TestValidate(testtools.TestCase): self.args.distro = 'centos7' self.args.stream = False self.args.no_stream = False - self.cmd = GenerateRepos(None, None) def test_good(self): - self.cmd._validate_args(self.args, '') + main._validate_args(self.args, '') def test_current_and_tripleo_dev(self): self.args.repos = ['current', 'current-tripleo-dev'] - self.assertRaises(E.InvalidArguments, self.cmd._validate_args, + self.assertRaises(main.InvalidArguments, main._validate_args, self.args, '') def test_tripleo_ci_testing_and_current_tripleo(self): self.args.repos = ['current-tripleo', 'tripleo-ci-testing'] - self.assertRaises(E.InvalidArguments, self.cmd._validate_args, + self.assertRaises(main.InvalidArguments, main._validate_args, self.args, '') def test_tripleo_ci_testing_and_ceph_opstools_allowed(self): self.args.repos = ['ceph', 'opstools', 'tripleo-ci-testing'] - self.cmd._validate_args(self.args, '') + main._validate_args(self.args, '') def test_tripleo_ci_testing_and_deps_allowed(self): self.args.repos = ['deps', 'tripleo-ci-testing'] - self.cmd._validate_args(self.args, '') + main._validate_args(self.args, '') def test_ceph_and_tripleo_dev(self): self.args.repos = ['current-tripleo-dev', 'ceph'] - self.args.output_path = C.DEFAULT_OUTPUT_PATH - self.cmd._validate_args(self.args, '') + self.args.output_path = main.DEFAULT_OUTPUT_PATH + main._validate_args(self.args, '') def test_deps_and_tripleo_dev(self): self.args.repos = ['deps', 'current-tripleo-dev'] - self.assertRaises(E.InvalidArguments, self.cmd._validate_args, + self.assertRaises(main.InvalidArguments, main._validate_args, self.args, '') def test_current_and_tripleo(self): self.args.repos = ['current', 'current-tripleo'] - self.assertRaises(E.InvalidArguments, self.cmd._validate_args, + self.assertRaises(main.InvalidArguments, main._validate_args, self.args, '') def test_deps_and_tripleo_allowed(self): self.args.repos = ['deps', 'current-tripleo'] - self.cmd._validate_args(self.args, '') + main._validate_args(self.args, '') def test_invalid_distro(self): self.args.distro = 'Jigawatts 1.21' - self.assertRaises(E.InvalidArguments, self.cmd._validate_args, + self.assertRaises(main.InvalidArguments, main._validate_args, self.args, '') def test_invalid_stream(self): self.args.stream = True - self.assertRaises(E.InvalidArguments, self.cmd._validate_args, + self.assertRaises(main.InvalidArguments, main._validate_args, self.args, 'CentOS 8') def test_invalid_no_stream(self): self.args.stream = False self.args.no_stream = True - self.assertRaises(E.InvalidArguments, self.cmd._validate_args, + self.assertRaises(main.InvalidArguments, main._validate_args, self.args, 'CentOS 8 Stream') def test_validate_distro_repos(self): - self.assertTrue(self.cmd._validate_distro_repos(self.args)) + self.assertTrue(main._validate_distro_repos(self.args)) def test_validate_distro_repos_fedora_tripleo_dev(self): self.args.distro = 'fedora' self.args.repos = ['current-tripleo-dev'] - self.assertRaises(E.InvalidArguments, self.cmd._validate_distro_repos, + self.assertRaises(main.InvalidArguments, main._validate_distro_repos, self.args)