From d0b144b0b21cc3fd316f360518695a99d79e60b1 Mon Sep 17 00:00:00 2001 From: Bulat Gaifullin Date: Wed, 11 Nov 2015 18:10:45 +0300 Subject: [PATCH] Introduced fuel-mirror Avaliable commands: - create - Creates a new local mirrors. fuel-mirror create --input ubuntu.yaml - apply - Applies local mirrors for Fuel-environments. fuel-mirror apply --input centos.yaml -e "ENV1" Change-Id: I6cbf4e581a93b47cf75fb3c89aa1020a352784c8 Closes-Bug: #1487077 Implements: blueprint refactor-local-mirror-scripts --- contrib/fuel_mirror/MANIFEST.in | 8 + contrib/fuel_mirror/README.rst | 16 + contrib/fuel_mirror/babel.cfg | 2 + contrib/fuel_mirror/data/centos.yaml | 58 ++++ contrib/fuel_mirror/data/ubuntu.yaml | 140 ++++++++ contrib/fuel_mirror/doc/source/conf.py | 75 +++++ .../fuel_mirror/doc/source/contributing.rst | 4 + contrib/fuel_mirror/doc/source/index.rst | 25 ++ .../fuel_mirror/doc/source/installation.rst | 12 + contrib/fuel_mirror/doc/source/readme.rst | 1 + contrib/fuel_mirror/doc/source/usage.rst | 7 + contrib/fuel_mirror/etc/config.yaml | 11 + contrib/fuel_mirror/fuel_mirror/__init__.py | 22 ++ contrib/fuel_mirror/fuel_mirror/app.py | 143 ++++++++ .../fuel_mirror/commands/__init__.py | 0 .../fuel_mirror/fuel_mirror/commands/apply.py | 158 +++++++++ .../fuel_mirror/fuel_mirror/commands/base.py | 97 ++++++ .../fuel_mirror/commands/create.py | 75 +++++ .../fuel_mirror/common/__init__.py | 0 .../fuel_mirror/common/accessors.py | 54 +++ .../fuel_mirror/common/url_builder.py | 68 ++++ .../fuel_mirror/fuel_mirror/common/utils.py | 96 ++++++ .../fuel_mirror/fuel_mirror/tests/__init__.py | 0 contrib/fuel_mirror/fuel_mirror/tests/base.py | 24 ++ .../fuel_mirror/tests/data/test_centos.yaml | 22 ++ .../fuel_mirror/tests/data/test_config.yaml | 6 + .../fuel_mirror/tests/data/test_ubuntu.yaml | 26 ++ .../fuel_mirror/tests/test_accessors.py | 90 +++++ .../fuel_mirror/tests/test_cli_commands.py | 317 ++++++++++++++++++ .../fuel_mirror/tests/test_url_builder.py | 68 ++++ .../fuel_mirror/tests/test_utils.py | 106 ++++++ .../fuel_mirror/tests/test_validate_config.py | 33 ++ contrib/fuel_mirror/requirements.txt | 9 + contrib/fuel_mirror/scripts/fuel-createmirror | 79 +++++ contrib/fuel_mirror/setup.cfg | 61 ++++ contrib/fuel_mirror/setup.py | 28 ++ contrib/fuel_mirror/test-requirements.txt | 17 + contrib/fuel_mirror/tox.ini | 35 ++ packetary/drivers/deb_driver.py | 18 +- packetary/drivers/rpm_driver.py | 22 +- packetary/library/utils.py | 10 + packetary/tests/test_deb_driver.py | 22 +- packetary/tests/test_rpm_driver.py | 19 +- 43 files changed, 2043 insertions(+), 41 deletions(-) create mode 100644 contrib/fuel_mirror/MANIFEST.in create mode 100644 contrib/fuel_mirror/README.rst create mode 100644 contrib/fuel_mirror/babel.cfg create mode 100644 contrib/fuel_mirror/data/centos.yaml create mode 100644 contrib/fuel_mirror/data/ubuntu.yaml create mode 100644 contrib/fuel_mirror/doc/source/conf.py create mode 100644 contrib/fuel_mirror/doc/source/contributing.rst create mode 100644 contrib/fuel_mirror/doc/source/index.rst create mode 100644 contrib/fuel_mirror/doc/source/installation.rst create mode 100644 contrib/fuel_mirror/doc/source/readme.rst create mode 100644 contrib/fuel_mirror/doc/source/usage.rst create mode 100644 contrib/fuel_mirror/etc/config.yaml create mode 100644 contrib/fuel_mirror/fuel_mirror/__init__.py create mode 100644 contrib/fuel_mirror/fuel_mirror/app.py create mode 100644 contrib/fuel_mirror/fuel_mirror/commands/__init__.py create mode 100644 contrib/fuel_mirror/fuel_mirror/commands/apply.py create mode 100644 contrib/fuel_mirror/fuel_mirror/commands/base.py create mode 100644 contrib/fuel_mirror/fuel_mirror/commands/create.py create mode 100644 contrib/fuel_mirror/fuel_mirror/common/__init__.py create mode 100644 contrib/fuel_mirror/fuel_mirror/common/accessors.py create mode 100644 contrib/fuel_mirror/fuel_mirror/common/url_builder.py create mode 100644 contrib/fuel_mirror/fuel_mirror/common/utils.py create mode 100644 contrib/fuel_mirror/fuel_mirror/tests/__init__.py create mode 100644 contrib/fuel_mirror/fuel_mirror/tests/base.py create mode 100644 contrib/fuel_mirror/fuel_mirror/tests/data/test_centos.yaml create mode 100644 contrib/fuel_mirror/fuel_mirror/tests/data/test_config.yaml create mode 100644 contrib/fuel_mirror/fuel_mirror/tests/data/test_ubuntu.yaml create mode 100644 contrib/fuel_mirror/fuel_mirror/tests/test_accessors.py create mode 100644 contrib/fuel_mirror/fuel_mirror/tests/test_cli_commands.py create mode 100644 contrib/fuel_mirror/fuel_mirror/tests/test_url_builder.py create mode 100644 contrib/fuel_mirror/fuel_mirror/tests/test_utils.py create mode 100644 contrib/fuel_mirror/fuel_mirror/tests/test_validate_config.py create mode 100644 contrib/fuel_mirror/requirements.txt create mode 100755 contrib/fuel_mirror/scripts/fuel-createmirror create mode 100644 contrib/fuel_mirror/setup.cfg create mode 100644 contrib/fuel_mirror/setup.py create mode 100644 contrib/fuel_mirror/test-requirements.txt create mode 100644 contrib/fuel_mirror/tox.ini diff --git a/contrib/fuel_mirror/MANIFEST.in b/contrib/fuel_mirror/MANIFEST.in new file mode 100644 index 0000000..5594774 --- /dev/null +++ b/contrib/fuel_mirror/MANIFEST.in @@ -0,0 +1,8 @@ +include AUTHORS +include ChangeLog +recursive-include etc * + +exclude .gitignore +exclude .gitreview + +global-exclude *.pyc diff --git a/contrib/fuel_mirror/README.rst b/contrib/fuel_mirror/README.rst new file mode 100644 index 0000000..7f67615 --- /dev/null +++ b/contrib/fuel_mirror/README.rst @@ -0,0 +1,16 @@ +=========== +fuel_mirror +=========== + +The fuel-mirror is utility, that allows to create local repositories +with packages are required for the OpenStack deployment. + +* Free software: Apache license +* Documentation: http://docs.openstack.org/developer/fuel-mirror +* Source: http://git.openstack.org/cgit/openstack/fuel-mirror/ +* Bugs: http://bugs.launchpad.net/fuel + +Features +-------- + +* TODO diff --git a/contrib/fuel_mirror/babel.cfg b/contrib/fuel_mirror/babel.cfg new file mode 100644 index 0000000..15cd6cb --- /dev/null +++ b/contrib/fuel_mirror/babel.cfg @@ -0,0 +1,2 @@ +[python: **.py] + diff --git a/contrib/fuel_mirror/data/centos.yaml b/contrib/fuel_mirror/data/centos.yaml new file mode 100644 index 0000000..6b9b993 --- /dev/null +++ b/contrib/fuel_mirror/data/centos.yaml @@ -0,0 +1,58 @@ +centos_baseurl: ¢os_baseurl +mos_baseurl: &mos_baseurl + +fuel_release_match: + version: $openstack_version + operating_system: CentOS + +repos: + - ¢os + name: "centos" + uri: "http://mirror.centos.org/centos/6/os/x86_64" + type: "rpm" + priority: null + + - ¢os_updates + name: "centos-updates" + uri: "http://mirror.centos.org/centos/6/updates/x86_64" + type: "rpm" + priority: null + + - &mos + name: "mos" + uri: "http://mirror.fuel-infra.org/mos-repos/centos/mos$mos_version-centos6-fuel/os/x86_64" + type: "rpm" + priority: null + + - &mos_updates + name: "mos-updates" + uri: "http://mirror.fuel-infra.org/mos-repos/centos/mos$mos_version-centos6-fuel/updates/x86_64" + type: "rpm" + priority: null + + - &mos_security + name: "mos-security" + uri: "http://mirror.fuel-infra.org/mos-repos/centos/mos$mos_version-centos6-fuel/security/x86_64" + type: "rpm" + priority: null + + - &mos_holdback + name: "mos-holdback" + uri: "http://mirror.fuel-infra.org/mos-repos/centos/mos$mos_version-centos6-fuel/holdback/x86_64" + type: "rpm" + priority: null + +groups: + mos: + - *mos + - *mos_updates + - *mos_security + - *mos_holdback + + centos: + - *centos + - *centos_updates + + +inheritance: + centos: mos diff --git a/contrib/fuel_mirror/data/ubuntu.yaml b/contrib/fuel_mirror/data/ubuntu.yaml new file mode 100644 index 0000000..9ffab1d --- /dev/null +++ b/contrib/fuel_mirror/data/ubuntu.yaml @@ -0,0 +1,140 @@ +# GLOBAL variables +ubuntu_baseurl: &ubuntu_baseurl http://archive.ubuntu.com/ubuntu +mos_baseurl: &mos_baseurl http://mirror.fuel-infra.org/mos-repos/ubuntu/$mos_version + +fuel_release_match: + version: $openstack_version + operating_system: Ubuntu + +repos: + - &ubuntu + name: "ubuntu" + uri: *ubuntu_baseurl + suite: "trusty" + section: "main multiverse restricted universe" + type: "deb" + priority: null + + - &ubuntu_updates + name: "ubuntu-updates" + uri: *ubuntu_baseurl + suite: "trusty-updates" + section: "main multiverse restricted universe" + type: "deb" + priority: null + + - &ubuntu_security + name: "ubuntu-security" + uri: *ubuntu_baseurl + suite: "trusty-security" + section: "main multiverse restricted universe" + type: "deb" + priority: null + + - &mos + name: "mos" + uri: *mos_baseurl + suite: "mos$mos_version" + section: "main restricted" + type: "deb" + priority: 1000 + + - &mos_updates + name: "mos-updates" + uri: *mos_baseurl + suite: "mos$mos_version-updates" + section: "main restricted" + type: "deb" + priority: 1000 + + - &mos_security + name: "mos-security" + uri: *mos_baseurl + suite: "mos$mos_version-security" + section: "main restricted" + type: "deb" + priority: 1000 + + - &mos_holdback + name: "mos-holdback" + uri: *mos_baseurl + suite: "mos$mos_version" + section: "main restricted" + type: "deb" + priority: 1000 + + +packages: &packages + - "acpi-support" + - "anacron" + - "aptitude" + - "atop" + - "bash-completion" + - "bc" + - "build-essential" + - "cloud-init" + - "conntrackd" + - "cpu-checker" + - "cpufrequtils" + - "debconf-utils" + - "devscripts" + - "fping" + - "git" + - "grub-pc" + - "htop" + - "ifenslave" + - "iperf" + - "iptables-persistent" + - "irqbalance" + - "language-pack-en" + - "linux-firmware-nonfree" + - "linux-headers-generic-lts-trusty" + - "linux-image-generic-lts-trusty" + - "livecd-rootfs" + - "memcached" + - "monit" + - "nginx" + - "ntp" + - "openssh-server" + - "percona-toolkit" + - "percona-xtrabackup" + - "pm-utils" + - "python-lesscpy" + - "python-pip" + - "puppet" + - "rsyslog-gnutls" + - "rsyslog-relp" + - "screen" + - "swift-plugin-s3" + - "sysfsutils" + - "sysstat" + - "telnet" + - "tmux" + - "traceroute" + - "ubuntu-standard" + - "vim" + - "virt-what" + - "xinetd" + + +groups: + mos: + - *mos + - *mos_updates + - *mos_security + - *mos_holdback + + ubuntu: + - *ubuntu + - *ubuntu_updates + - *ubuntu_security + + +inheritance: + ubuntu: mos + +osnames: + mos: ubuntu + +requirements: + ubuntu: *packages diff --git a/contrib/fuel_mirror/doc/source/conf.py b/contrib/fuel_mirror/doc/source/conf.py new file mode 100644 index 0000000..9491581 --- /dev/null +++ b/contrib/fuel_mirror/doc/source/conf.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + #'sphinx.ext.intersphinx', + 'oslosphinx' +] + +# autodoc generation is a bit aggressive and a nuisance when doing heavy +# text edit cycles. +# execute "export SPHINX_DEBUG=1" in your terminal to disable + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'fuel_mirror' +copyright = u'2015, Mirantis, Inc' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +# html_theme = '_theme' +# html_static_path = ['static'] + +# Output file base name for HTML help builder. +htmlhelp_basename = '%sdoc' % project + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +latex_documents = [ + ('index', + '%s.tex' % project, + u'%s Documentation' % project, + u'OpenStack Foundation', 'manual'), +] + +# Example configuration for intersphinx: refer to the Python standard library. +#intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/contrib/fuel_mirror/doc/source/contributing.rst b/contrib/fuel_mirror/doc/source/contributing.rst new file mode 100644 index 0000000..36570d8 --- /dev/null +++ b/contrib/fuel_mirror/doc/source/contributing.rst @@ -0,0 +1,4 @@ +============ +Contributing +============ +.. include:: ../../../../CONTRIBUTING.rst diff --git a/contrib/fuel_mirror/doc/source/index.rst b/contrib/fuel_mirror/doc/source/index.rst new file mode 100644 index 0000000..2be2378 --- /dev/null +++ b/contrib/fuel_mirror/doc/source/index.rst @@ -0,0 +1,25 @@ +.. fuel_mirror documentation master file, created by + sphinx-quickstart on Tue Jul 9 22:26:36 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to fuel_mirror's documentation! +======================================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + readme + installation + usage + contributing + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/contrib/fuel_mirror/doc/source/installation.rst b/contrib/fuel_mirror/doc/source/installation.rst new file mode 100644 index 0000000..be36bdf --- /dev/null +++ b/contrib/fuel_mirror/doc/source/installation.rst @@ -0,0 +1,12 @@ +============ +Installation +============ + +At the command line:: + + $ pip install fuel_mirror + +Or, if you have virtualenvwrapper installed:: + + $ mkvirtualenv fuel_mirror + $ pip install fuel_mirror diff --git a/contrib/fuel_mirror/doc/source/readme.rst b/contrib/fuel_mirror/doc/source/readme.rst new file mode 100644 index 0000000..a6210d3 --- /dev/null +++ b/contrib/fuel_mirror/doc/source/readme.rst @@ -0,0 +1 @@ +.. include:: ../../README.rst diff --git a/contrib/fuel_mirror/doc/source/usage.rst b/contrib/fuel_mirror/doc/source/usage.rst new file mode 100644 index 0000000..a1879eb --- /dev/null +++ b/contrib/fuel_mirror/doc/source/usage.rst @@ -0,0 +1,7 @@ +======== +Usage +======== + +To use fuel_mirror in a project:: + + import fuel_createmirror diff --git a/contrib/fuel_mirror/etc/config.yaml b/contrib/fuel_mirror/etc/config.yaml new file mode 100644 index 0000000..3a92bd2 --- /dev/null +++ b/contrib/fuel_mirror/etc/config.yaml @@ -0,0 +1,11 @@ +threads_num: 10 +ignore_errors_num: 2 +retries_num: 3 +target_dir: "/var/www/nailgun/mirrors" +pattern_dir: "/usr/share/fuel-mirror" + +# uncomment if need +# http_proxy: null +# https_proxy: null +# fuel_server: 10.20.0.2 +# base_url: http://10.20.0.2:8080 diff --git a/contrib/fuel_mirror/fuel_mirror/__init__.py b/contrib/fuel_mirror/fuel_mirror/__init__.py new file mode 100644 index 0000000..d1422b9 --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import pbr.version + + +__version__ = pbr.version.VersionInfo( + 'fuel_mirror').version_string() diff --git a/contrib/fuel_mirror/fuel_mirror/app.py b/contrib/fuel_mirror/fuel_mirror/app.py new file mode 100644 index 0000000..17a1ce2 --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/app.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from cliff import app +from cliff.commandmanager import CommandManager +import yaml + + +import fuel_mirror +from fuel_mirror.common import accessors +from fuel_mirror.common import utils + + +_FUEL_DEFAULT_HTTP_PORT = 8080 + + +class Application(app.App): + """Main cliff application class. + + Performs initialization of the command manager and + configuration of basic engines. + """ + + config = None + fuel = None + repo_manager_accessor = None + sources = None + versions = None + + def build_option_parser(self, description, version, argparse_kwargs=None): + """Specifies common cmdline arguments.""" + p_inst = super(Application, self) + parser = p_inst.build_option_parser(description=description, + version=version, + argparse_kwargs=argparse_kwargs) + + parser.add_argument( + "--config", + default="/etc/fuel-mirror/config.yaml", + metavar="PATH", + help="Path to config file." + ) + parser.add_argument( + "-S", "--fuel-server", + metavar="FUEL-SERVER", + help="The public address of Fuel Master." + ) + parser.add_argument( + "--fuel-user", + help="Fuel Master admin login." + " Alternatively, use env var KEYSTONE_USER)." + ) + parser.add_argument( + "--fuel-password", + help="Fuel Master admin password." + " Alternatively, use env var KEYSTONE_PASSWORD)." + ) + return parser + + def initialize_app(self, argv): + """Initialises common options.""" + with open(self.options.config, "r") as stream: + self.config = yaml.load(stream) + fuel_default = utils.get_fuel_settings() + + fuel_server = utils.first( + self.options.fuel_server, + self.config.get("fuel_server"), + fuel_default["server"] + ) + fuel_user = utils.first( + self.options.fuel_user, + fuel_default["user"] + ) + fuel_password = utils.first( + self.options.fuel_password, + fuel_default["password"] + ) + self.config.setdefault( + "base_url", "http://{0}:{1}".format( + fuel_server.split(":", 1)[0], + _FUEL_DEFAULT_HTTP_PORT + ) + ) + self.fuel = accessors.get_fuel_api_accessor( + fuel_server, + fuel_user, + fuel_password + ) + fuel_ver = self.fuel.FuelVersion.get_all_data() + self.config.setdefault( + 'mos_version', fuel_ver['release'] + ) + self.config.setdefault( + 'openstack_version', fuel_ver['openstack_version'] + ) + + self.repo_manager_accessor = accessors.get_packetary_accessor( + threads_num=int(self.config.get('threads_num', 0)), + retries_num=int(self.config.get('retries_num', 0)), + ignore_errors_num=int(self.config.get('ignore_errors_num', 0)), + http_proxy=self.config.get('http_proxy'), + https_proxy=self.config.get('https_proxy'), + ) + + +def main(argv=None): + """Entry point.""" + return Application( + description="The utility to create local mirrors.", + version=fuel_mirror.__version__, + command_manager=CommandManager("fuel_mirror", convert_underscores=True) + ).run(argv) + + +def debug(name, cmd_class, argv=None): + """Helps to debug command.""" + import sys + + if argv is None: + argv = sys.argv[1:] + + argv = [name] + argv + ["-v", "-v", "--debug"] + cmd_mgr = CommandManager("test_fuel_mirror", convert_underscores=True) + cmd_mgr.add_command(name, cmd_class) + return Application( + description="The fuel mirror utility test.", + version="0.0.1", + command_manager=cmd_mgr + ).run(argv) diff --git a/contrib/fuel_mirror/fuel_mirror/commands/__init__.py b/contrib/fuel_mirror/fuel_mirror/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/contrib/fuel_mirror/fuel_mirror/commands/apply.py b/contrib/fuel_mirror/fuel_mirror/commands/apply.py new file mode 100644 index 0000000..cd511a9 --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/commands/apply.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import six + +from packetary.library.utils import localize_repo_url + +from fuel_mirror.commands.base import BaseCommand +from fuel_mirror.common.utils import is_subdict +from fuel_mirror.common.utils import lists_merge + + +class ApplyCommand(BaseCommand): + """Applies local mirrors for Fuel-environments.""" + + def get_parser(self, prog_name): + parser = super(ApplyCommand, self).get_parser(prog_name) + parser.add_argument( + "--default", + dest="set_default", + action="store_true", + default=False, + help="Set as default repository." + ) + parser.add_argument( + "-e", "--env", + dest="env", nargs="+", + help="Fuel environment ID to update, " + "by default applies for all environments." + ) + + return parser + + def take_action(self, parsed_args): + data = self.load_data(parsed_args) + base_url = self.app.config["base_url"] + localized_repos = [] + for _, repos in self.get_groups(parsed_args, data): + for repo_data in repos: + new_data = repo_data.copy() + new_data['uri'] = localize_repo_url( + base_url, repo_data['uri'] + ) + localized_repos.append(new_data) + + release_match = data["fuel_release_match"] + self.update_clusters(parsed_args.env, localized_repos, release_match) + if parsed_args.set_default: + self.update_default_repos(localized_repos, release_match) + + self.app.stdout.write( + "Operations have been completed successfully.\n" + ) + + def update_clusters(self, ids, repositories, release_match): + """Applies repositories for existing clusters. + + :param ids: the cluster ids. + :param repositories: the meta information of repositories + :param release_match: The pattern to check Fuel Release + """ + self.app.stdout.write("Updating the Cluster repositories...\n") + + if ids: + clusters = self.app.fuel.Environment.get_by_ids(ids) + else: + clusters = self.app.fuel.Environment.get_all() + + for cluster in clusters: + releases = six.moves.filter( + lambda x: is_subdict(release_match, x.data), + self.app.fuel.Release.get_by_ids([cluster.data["release_id"]]) + ) + if next(releases, None) is None: + continue + + modified = self._update_repository_settings( + cluster.get_settings_data(), + repositories + ) + if modified: + self.app.LOG.info( + "Try to update the Cluster '%s'", + cluster.data['name'] + ) + self.app.LOG.debug( + "The modified cluster attributes: %s", + modified + ) + cluster.set_settings_data(modified) + + def update_default_repos(self, repositories, release_match): + """Applies repositories for existing default settings. + + :param repositories: the meta information of repositories + :param release_match: The pattern to check Fuel Release + """ + self.app.stdout.write("Updating the default repositories...\n") + releases = six.moves.filter( + lambda x: is_subdict(release_match, x.data), + self.app.fuel.Release.get_all() + ) + for release in releases: + if self._update_repository_settings( + release.data["attributes_metadata"], repositories + ): + self.app.LOG.info( + "Try to update the Release '%s'", + release.data['name'] + ) + self.app.LOG.debug( + "The modified release attributes: %s", + release.data + ) + # TODO(need to add method for release object) + release.connection.put_request( + release.instance_api_path.format(release.id), + release.data + ) + + def _update_repository_settings(self, settings, repositories): + """Updates repository settings. + + :param settings: the target settings + :param repositories: the meta of repositories + """ + editable = settings["editable"] + if 'repo_setup' not in editable: + self.app.LOG.info('Attributes is read-only.') + return + + repos_attr = editable["repo_setup"]["repos"] + lists_merge(repos_attr['value'], repositories, "name") + return {"editable": {"repo_setup": {"repos": repos_attr}}} + + +def debug(argv=None): + """Helper for debugging Apply command.""" + from fuel_mirror.app import debug + + debug("apply", ApplyCommand, argv) + + +if __name__ == "__main__": + debug() diff --git a/contrib/fuel_mirror/fuel_mirror/commands/base.py b/contrib/fuel_mirror/fuel_mirror/commands/base.py new file mode 100644 index 0000000..6f7c1ff --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/commands/base.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os.path +from string import Template + +from cliff import command +import yaml + + +class BaseCommand(command.Command): + """The Base command for fuel-mirror.""" + REPO_ARCH = "x86_64" + + @property + def stdout(self): + """Shortcut for self.app.stdout.""" + return self.app.stdout + + def get_parser(self, prog_name): + """Specifies common options.""" + parser = super(BaseCommand, self).get_parser(prog_name) + + input_group = parser.add_mutually_exclusive_group(required=True) + input_group.add_argument( + '-I', '--input-file', + metavar='PATH', + help='The path to file with input data.') + + input_group.add_argument( + '-P', '--pattern', + metavar='NAME', + help='The builtin input file name.' + ) + + parser.add_argument( + "-G", "--group", + dest="groups", + required=True, + nargs='+', + help="The name of repository groups." + ) + return parser + + def resolve_input_pattern(self, pattern): + """Gets the full path to input file by pattern. + + :param pattern: the config file name without ext + :return: the full path + """ + return os.path.join( + self.app.config['pattern_dir'], pattern + ".yaml" + ) + + def load_data(self, parsed_args): + """Load the input data. + + :param parsed_args: the command-line arguments + :return: the input data + """ + if parsed_args.pattern: + input_file = self.resolve_input_pattern(parsed_args.pattern) + else: + input_file = parsed_args.input_file + + # TODO(add input data validation scheme) + with open(input_file, "r") as fd: + return yaml.load(Template(fd.read()).safe_substitute( + mos_version=self.app.config["mos_version"], + openstack_version=self.app.config["openstack_version"], + )) + + @classmethod + def get_groups(cls, parsed_args, data): + """Gets repository groups from input data. + + :param parsed_args: the command-line arguments + :param data: the input data + :return: the sequence of pairs (group_name, repositories) + """ + all_groups = data['groups'] + return ( + (x, all_groups[x]) for x in parsed_args.groups if x in all_groups + ) diff --git a/contrib/fuel_mirror/fuel_mirror/commands/create.py b/contrib/fuel_mirror/fuel_mirror/commands/create.py new file mode 100644 index 0000000..846b1df --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/commands/create.py @@ -0,0 +1,75 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fuel_mirror.commands.base import BaseCommand +from fuel_mirror.common.url_builder import get_url_builder + + +class CreateCommand(BaseCommand): + """Creates a new local mirrors.""" + + def take_action(self, parsed_args): + """See the Command.take_action.""" + data = self.load_data(parsed_args) + repos_reqs = data.get('requirements', {}) + inheritance = data.get('inheritance', {}) + target_dir = self.app.config["target_dir"] + + total_stats = None + for group_name, repos in self.get_groups(parsed_args, data): + url_builder = get_url_builder(repos[0]["type"]) + repo_manager = self.app.repo_manager_accessor( + repos[0]["type"], self.REPO_ARCH + ) + if group_name in inheritance: + child_group = inheritance[group_name] + dependencies = [ + url_builder.get_repo_url(x) + for x in data['groups'][child_group] + ] + else: + dependencies = None + + stat = repo_manager.clone_repositories( + [url_builder.get_repo_url(x) for x in repos], + target_dir, + dependencies, + repos_reqs.get(group_name) + ) + + if total_stats is None: + total_stats = stat + else: + total_stats += stat + + if total_stats is not None: + self.stdout.write( + "Packages processed: {0.copied}/{0.total}\n" + .format(total_stats) + ) + else: + self.stdout.write( + "No packages.\n" + ) + + +def debug(argv=None): + """Helper for debugging Create command.""" + from fuel_mirror.app import debug + + debug("create", CreateCommand, argv) + + +if __name__ == "__main__": + debug() diff --git a/contrib/fuel_mirror/fuel_mirror/common/__init__.py b/contrib/fuel_mirror/fuel_mirror/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/contrib/fuel_mirror/fuel_mirror/common/accessors.py b/contrib/fuel_mirror/fuel_mirror/common/accessors.py new file mode 100644 index 0000000..e2cbc0a --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/common/accessors.py @@ -0,0 +1,54 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import functools +import os + + +def get_packetary_accessor(**kwargs): + """Gets the configured repository manager. + + :param kwargs: The packetary configuration parameters. + """ + + import packetary + + return functools.partial( + packetary.RepositoryApi.create, + packetary.Context(packetary.Configuration(**kwargs)) + ) + + +def get_fuel_api_accessor(address=None, user=None, password=None): + """Gets the fuel client api accessor. + + :param address: The address of Fuel Master node. + :param user: The username to access to the Fuel Master node. + :param user: The password to access to the Fuel Master node. + """ + if address: + host_and_port = address.split(":") + os.environ["SERVER_ADDRESS"] = host_and_port[0] + if len(host_and_port) > 1: + os.environ["LISTEN_PORT"] = host_and_port[1] + + if user is not None: + os.environ["KEYSTONE_USER"] = user + if password is not None: + os.environ["KEYSTONE_PASS"] = password + + # import fuelclient.ClientAPI after configuring + # environment variables + from fuelclient import objects + return objects diff --git a/contrib/fuel_mirror/fuel_mirror/common/url_builder.py b/contrib/fuel_mirror/fuel_mirror/common/url_builder.py new file mode 100644 index 0000000..685c393 --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/common/url_builder.py @@ -0,0 +1,68 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def get_url_builder(repotype): + """Gets the instance of RepoUrlBuilder. + + :param repotype: the type of repository: rpm|deb + :return: the RepoBuilder implementation + """ + return { + "deb": AptRepoUrlBuilder, + "rpm": YumRepoUrlBuilder + }[repotype] + + +class RepoUrlBuilder(object): + REPO_FOLDER = "mirror" + + @classmethod + def get_repo_url(cls, repo_data): + """Gets the url with replaced variable holders. + + :param repo_data: the repositories`s meta data + :return: the full repository`s url + """ + + +class AptRepoUrlBuilder(RepoUrlBuilder): + """URL builder for apt-repository(es).""" + + @classmethod + def get_repo_url(cls, repo_data): + return " ".join( + repo_data[x] for x in ("uri", "suite", "section") + ) + + +class YumRepoUrlBuilder(RepoUrlBuilder): + """URL builder for Yum repository(es).""" + + @classmethod + def split_url(cls, url, maxsplit=2): + """Splits url to baseurl, reponame adn architecture. + + :param url: the repository`s URL + :param maxsplit: the number of expected components + :return the components of url + """ + # TODO(need generic url building algorithm) + # there is used assumption that url has following format + # $baseurl/$reponame/$repoarch + return url.rstrip("/").rsplit("/", maxsplit) + + @classmethod + def get_repo_url(cls, repo_data): + return cls.split_url(repo_data["uri"], 1)[0] diff --git a/contrib/fuel_mirror/fuel_mirror/common/utils.py b/contrib/fuel_mirror/fuel_mirror/common/utils.py new file mode 100644 index 0000000..44f48d4 --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/common/utils.py @@ -0,0 +1,96 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import six +import yaml + + +def lists_merge(main, patch, key): + """Merges the list of dicts with same keys. + + >>> lists_merge([{"a": 1, "c": 2}], [{"a": 1, "c": 3}], key="a") + [{'a': 1, 'c': 3}] + + :param main: the main list + :type main: list + :param patch: the list of additional elements + :type patch: list + :param key: the key for compare + """ + main_idx = dict( + (x[key], i) for i, x in enumerate(main) + ) + + patch_idx = dict( + (x[key], i) for i, x in enumerate(patch) + ) + + for k in sorted(patch_idx): + if k in main_idx: + main[main_idx[k]].update(patch[patch_idx[k]]) + else: + main.append(patch[patch_idx[k]]) + return main + + +def is_subdict(dict1, dict2): + """Checks that dict1 is subdict of dict2. + + >>> is_subdict({"a": 1}, {'a': 1, 'b': 1}) + True + + :param dict1: the candidate + :param dict2: the super dict + :return: True if all keys from dict1 are present + and has same value in dict2 otherwise False + """ + for k, v in six.iteritems(dict1): + if k not in dict2 or dict2[k] != v: + return False + return True + + +def first(*args): + """Get first not empty value. + + >>> first(0, 1) == next(iter(filter(None, [0, 1]))) + True + + :param args: the list of arguments + :return first value that bool(v) is True, None if not found. + """ + for arg in args: + if arg: + return arg + + +def get_fuel_settings(): + """Gets the fuel settings from astute container, if it is available.""" + + _DEFAULT_SETTINGS = { + "server": "10.20.0.2", + "user": None, + "password": None, + } + try: + with open("/etc/fuel/astute.yaml", "r") as fd: + settings = yaml.load(fd) + return { + "server": settings.get("ADMIN_NETWORK", {}).get("ipaddress"), + "user": settings.get("FUEL_ACCESS", {}).get("user"), + "password": settings.get("FUEL_ACCESS", {}).get("password") + } + except (OSError, IOError): + pass + return _DEFAULT_SETTINGS diff --git a/contrib/fuel_mirror/fuel_mirror/tests/__init__.py b/contrib/fuel_mirror/fuel_mirror/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/contrib/fuel_mirror/fuel_mirror/tests/base.py b/contrib/fuel_mirror/fuel_mirror/tests/base.py new file mode 100644 index 0000000..51c9a93 --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/tests/base.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +try: + import unittest2 as unittest +except ImportError: + import unittest + + +class TestCase(unittest.TestCase): + """Test case base class for all unit tests.""" diff --git a/contrib/fuel_mirror/fuel_mirror/tests/data/test_centos.yaml b/contrib/fuel_mirror/fuel_mirror/tests/data/test_centos.yaml new file mode 100644 index 0000000..11f69bf --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/tests/data/test_centos.yaml @@ -0,0 +1,22 @@ +fuel_release_match: + operating_system: CentOS + +inheritance: + centos: mos + +groups: + mos: + - name: "mos" + type: "rpm" + uri: "http://localhost/mos$mos_version/x86_64" + priority: 10 + + centos: + - name: "centos" + type: "rpm" + uri: "http://localhost/centos/os/x86_64" + priority: 5 + +requirements: + centos: + - "package_rpm" diff --git a/contrib/fuel_mirror/fuel_mirror/tests/data/test_config.yaml b/contrib/fuel_mirror/fuel_mirror/tests/data/test_config.yaml new file mode 100644 index 0000000..94be04d --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/tests/data/test_config.yaml @@ -0,0 +1,6 @@ +threads_num: 1 +ignore_errors_num: 2 +retries_num: 3 +http_proxy: "http://localhost" +https_proxy: "https://localhost" +target_dir: "/var/www/" \ No newline at end of file diff --git a/contrib/fuel_mirror/fuel_mirror/tests/data/test_ubuntu.yaml b/contrib/fuel_mirror/fuel_mirror/tests/data/test_ubuntu.yaml new file mode 100644 index 0000000..5ee58fc --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/tests/data/test_ubuntu.yaml @@ -0,0 +1,26 @@ +fuel_release_match: + operating_system: Ubuntu + +inheritance: + ubuntu: mos + +groups: + mos: + - name: "mos" + type: "deb" + uri: "http://localhost/mos" + suite: "mos$mos_version" + section: "main restricted" + priority: 1000 + + ubuntu: + - name: "ubuntu" + type: "deb" + uri: "http://localhost/ubuntu" + suite: "trusty" + section: "main multiverse restricted universe" + priority: 500 + +requirements: + ubuntu: + - "package_deb" diff --git a/contrib/fuel_mirror/fuel_mirror/tests/test_accessors.py b/contrib/fuel_mirror/fuel_mirror/tests/test_accessors.py new file mode 100644 index 0000000..dfd78fa --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/tests/test_accessors.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from fuel_mirror.common import accessors +from fuel_mirror.tests import base + + +class TestAccessors(base.TestCase): + def test_get_packetary_accessor(self): + packetary = mock.MagicMock() + with mock.patch.dict("sys.modules", packetary=packetary): + accessor = accessors.get_packetary_accessor( + http_proxy="http://localhost", + https_proxy="https://localhost", + retries_num=1, + threads_num=2, + ignore_errors_num=3 + ) + accessor("deb") + accessor("yum") + packetary.Configuration.assert_called_once_with( + http_proxy="http://localhost", + https_proxy="https://localhost", + retries_num=1, + threads_num=2, + ignore_errors_num=3 + ) + packetary.Context.assert_called_once_with( + packetary.Configuration() + ) + self.assertEqual(2, packetary.RepositoryApi.create.call_count) + packetary.RepositoryApi.create.assert_any_call( + packetary.Context(), "deb" + ) + packetary.RepositoryApi.create.assert_any_call( + packetary.Context(), "yum" + ) + + @mock.patch("fuel_mirror.common.accessors.os") + def test_get_fuel_api_accessor(self, os): + fuelclient = mock.MagicMock() + patch = { + "fuelclient": fuelclient, + "fuelclient.objects": fuelclient.objects + } + with mock.patch.dict("sys.modules", patch): + accessor = accessors.get_fuel_api_accessor( + "localhost:8080", "guest", "123" + ) + accessor.Environment.get_all() + + os.environ.__setitem__.asseert_any_call( + "SERVER_ADDRESS", "localhost" + ) + os.environ.__setitem__.asseert_any_call( + "LISTEN_PORT", "8080" + ) + os.environ.__setitem__.asseert_any_call( + "KEYSTONE_USER", "guest" + ) + os.environ.__setitem__.asseert_any_call( + "KEYSTONE_PASS", "123" + ) + fuelclient.objects.Environment.get_all.assert_called_once_with() + + @mock.patch("fuel_mirror.common.accessors.os") + def test_get_fuel_api_accessor_with_default_parameters(self, os): + fuelclient = mock.MagicMock() + patch = { + "fuelclient": fuelclient, + "fuelclient.objects": fuelclient.objects + } + with mock.patch.dict("sys.modules", patch): + accessors.get_fuel_api_accessor() + os.environ.__setitem__.assert_not_called() diff --git a/contrib/fuel_mirror/fuel_mirror/tests/test_cli_commands.py b/contrib/fuel_mirror/fuel_mirror/tests/test_cli_commands.py new file mode 100644 index 0000000..09e8300 --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/tests/test_cli_commands.py @@ -0,0 +1,317 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import os.path +import subprocess + +# The cmd2 does not work with python3.5 +# because it tries to get access to the property mswindows, +# that was removed in 3.5 +subprocess.mswindows = False + +from fuel_mirror.commands import apply +from fuel_mirror.commands import create +from fuel_mirror.tests import base + + +CONFIG_PATH = os.path.join( + os.path.dirname(__file__), "data", "test_config.yaml" +) + +UBUNTU_PATH = os.path.join( + os.path.dirname(__file__), "data", "test_ubuntu.yaml" +) + +CENTOS_PATH = os.path.join( + os.path.dirname(__file__), "data", "test_centos.yaml" +) + + +@mock.patch.multiple( + "fuel_mirror.app", + accessors=mock.DEFAULT, +) +class TestCliCommands(base.TestCase): + common_argv = [ + "--config", CONFIG_PATH, + "--fuel-server=10.25.0.10", + "--fuel-user=test", + "--fuel-password=test1" + ] + + def start_cmd(self, cmd, argv, data_file): + cmd.debug( + argv + self.common_argv + ["--input-file", data_file] + ) + + def _setup_fuel_versions(self, fuel_mock): + fuel_mock.FuelVersion.get_all_data.return_value = { + "release": "1", + "openstack_version": "2" + } + + def _create_fuel_release(self, fuel_mock, osname): + release = mock.MagicMock(data={ + "name": "test release", + "operating_system": osname, + "attributes_metadata": { + "editable": {"repo_setup": {"repos": {"value": []}}} + } + }) + + fuel_mock.Release.get_by_ids.return_value = [release] + fuel_mock.Release.get_all.return_value = [release] + return release + + def _create_fuel_env(self, fuel_mock): + env = mock.MagicMock(data={ + "name": "test", + "release_id": 1 + }) + env.get_settings_data.return_value = { + "editable": {"repo_setup": {"repos": {"value": []}}} + } + fuel_mock.Environment.get_by_ids.return_value = [env] + fuel_mock.Environment.get_all.return_value = [env] + return env + + def test_create_mos_ubuntu(self, accessors): + self._setup_fuel_versions(accessors.get_fuel_api_accessor()) + packetary = accessors.get_packetary_accessor() + + self.start_cmd(create, ["--group", "mos"], UBUNTU_PATH) + accessors.get_packetary_accessor.assert_called_with( + threads_num=1, + ignore_errors_num=2, + retries_num=3, + http_proxy="http://localhost", + https_proxy="https://localhost", + ) + packetary.assert_called_with("deb", "x86_64") + api = packetary() + api.clone_repositories.assert_called_once_with( + ['http://localhost/mos mos1 main restricted'], + '/var/www/', + None, None + ) + + def test_create_partial_ubuntu(self, accessors): + self._setup_fuel_versions(accessors.get_fuel_api_accessor()) + packetary = accessors.get_packetary_accessor() + + self.start_cmd(create, ["--group", "ubuntu"], UBUNTU_PATH) + accessors.get_packetary_accessor.assert_called_with( + threads_num=1, + ignore_errors_num=2, + retries_num=3, + http_proxy="http://localhost", + https_proxy="https://localhost", + ) + packetary.assert_called_with("deb", "x86_64") + api = packetary() + api.clone_repositories.assert_called_once_with( + ['http://localhost/ubuntu trusty ' + 'main multiverse restricted universe'], + '/var/www/', + ['http://localhost/mos mos1 main restricted'], + ['package_deb'] + ) + + def test_create_mos_centos(self, accessors): + self._setup_fuel_versions(accessors.get_fuel_api_accessor()) + packetary = accessors.get_packetary_accessor() + + self.start_cmd(create, ["--group", "mos"], CENTOS_PATH) + accessors.get_packetary_accessor.assert_called_with( + threads_num=1, + ignore_errors_num=2, + retries_num=3, + http_proxy="http://localhost", + https_proxy="https://localhost", + ) + packetary.assert_called_with("rpm", "x86_64") + api = packetary() + api.clone_repositories.assert_called_once_with( + ['http://localhost/mos1'], + '/var/www/', + None, None + ) + + def test_create_partial_centos(self, accessors): + self._setup_fuel_versions(accessors.get_fuel_api_accessor()) + packetary = accessors.get_packetary_accessor() + + self.start_cmd(create, ["--group", "centos"], CENTOS_PATH) + accessors.get_packetary_accessor.assert_called_with( + threads_num=1, + ignore_errors_num=2, + retries_num=3, + http_proxy="http://localhost", + https_proxy="https://localhost", + ) + packetary.assert_called_with("rpm", "x86_64") + api = packetary() + api.clone_repositories.assert_called_once_with( + ['http://localhost/centos/os'], + '/var/www/', + ['http://localhost/mos1'], + ["package_rpm"] + ) + + def test_apply_for_ubuntu_based_env(self, accessors): + fuel = accessors.get_fuel_api_accessor() + self._setup_fuel_versions(fuel) + env = self._create_fuel_env(fuel) + self._create_fuel_release(fuel, "Ubuntu") + self.start_cmd( + apply, ['--group', 'mos', 'ubuntu', '--env', '1'], + UBUNTU_PATH + ) + accessors.get_fuel_api_accessor.assert_called_with( + "10.25.0.10", "test", "test1" + ) + fuel.FuelVersion.get_all_data.assert_called_once_with() + env.set_settings_data.assert_called_with( + {'editable': {'repo_setup': {'repos': {'value': [ + { + 'priority': 1000, + 'name': 'mos', + 'suite': 'mos1', + 'section': 'main restricted', + 'type': 'deb', + 'uri': 'http://10.25.0.10:8080/mos' + }, + { + 'priority': 500, + 'name': 'ubuntu', + 'suite': 'trusty', + 'section': 'main multiverse restricted universe', + 'type': 'deb', + 'uri': 'http://10.25.0.10:8080/ubuntu' + } + ]}}}} + ) + + def test_apply_for_centos_based_env(self, accessors): + fuel = accessors.get_fuel_api_accessor() + self._setup_fuel_versions(fuel) + env = self._create_fuel_env(fuel) + self._create_fuel_release(fuel, "CentOS") + self.start_cmd( + apply, ['--group', 'mos', 'centos', '--env', '1'], + CENTOS_PATH + ) + accessors.get_fuel_api_accessor.assert_called_with( + "10.25.0.10", "test", "test1" + ) + fuel.FuelVersion.get_all_data.assert_called_once_with() + env.set_settings_data.assert_called_with( + {'editable': {'repo_setup': {'repos': {'value': [ + { + 'priority': 5, + 'name': 'centos', + 'type': 'rpm', + 'uri': 'http://10.25.0.10:8080/centos/os/x86_64' + }, + { + 'priority': 10, + 'name': 'mos', + 'type': 'rpm', + 'uri': 'http://10.25.0.10:8080/mos1/x86_64' + }] + }}}} + ) + + def test_apply_for_ubuntu_release(self, accessors): + fuel = accessors.get_fuel_api_accessor() + self._setup_fuel_versions(fuel) + env = self._create_fuel_env(fuel) + release = self._create_fuel_release(fuel, "Ubuntu") + self.start_cmd( + apply, ['--group', 'mos', 'ubuntu', '--default'], + UBUNTU_PATH + ) + accessors.get_fuel_api_accessor.assert_called_with( + "10.25.0.10", "test", "test1" + ) + fuel.FuelVersion.get_all_data.assert_called_once_with() + self.assertEqual(1, env.set_settings_data.call_count) + release.connection.put_request.assert_called_once_with( + release.instance_api_path.format(), + { + 'name': "test release", + 'operating_system': 'Ubuntu', + 'attributes_metadata': { + 'editable': {'repo_setup': {'repos': {'value': [ + { + 'name': 'mos', + 'priority': 1000, + 'suite': 'mos1', + 'section': 'main restricted', + 'type': 'deb', + 'uri': 'http://10.25.0.10:8080/mos' + }, + { + 'name': 'ubuntu', + 'priority': 500, + 'suite': 'trusty', + 'section': 'main multiverse restricted universe', + 'type': 'deb', + 'uri': 'http://10.25.0.10:8080/ubuntu' + } + ]}}} + } + } + ) + + def test_apply_for_centos_release(self, accessors): + fuel = accessors.get_fuel_api_accessor() + self._setup_fuel_versions(fuel) + env = self._create_fuel_env(fuel) + release = self._create_fuel_release(fuel, "CentOS") + self.start_cmd( + apply, ['--group', 'mos', 'centos', '--default'], + CENTOS_PATH + ) + accessors.get_fuel_api_accessor.assert_called_with( + "10.25.0.10", "test", "test1" + ) + fuel.FuelVersion.get_all_data.assert_called_once_with() + self.assertEqual(1, env.set_settings_data.call_count) + release.connection.put_request.assert_called_once_with( + release.instance_api_path.format(), + { + 'name': "test release", + 'operating_system': 'CentOS', + 'attributes_metadata': { + 'editable': {'repo_setup': {'repos': {'value': [ + { + 'name': 'centos', + 'priority': 5, + 'type': 'rpm', + 'uri': 'http://10.25.0.10:8080/centos/os/x86_64' + }, + { + 'name': 'mos', + 'priority': 10, + 'type': 'rpm', + 'uri': 'http://10.25.0.10:8080/mos1/x86_64' + }, + ]}}} + } + } + ) diff --git a/contrib/fuel_mirror/fuel_mirror/tests/test_url_builder.py b/contrib/fuel_mirror/fuel_mirror/tests/test_url_builder.py new file mode 100644 index 0000000..dfcf1b4 --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/tests/test_url_builder.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fuel_mirror.common import url_builder +from fuel_mirror.tests import base + + +class TestUrlBuilder(base.TestCase): + def test_get_url_builder(self): + self.assertTrue(issubclass( + url_builder.get_url_builder("deb"), + url_builder.AptRepoUrlBuilder + )) + self.assertTrue(issubclass( + url_builder.get_url_builder("rpm"), + url_builder.YumRepoUrlBuilder + )) + with self.assertRaises(KeyError): + url_builder.get_url_builder("unknown") + + +class TestAptUrlBuilder(base.TestCase): + @classmethod + def setUpClass(cls): + cls.builder = url_builder.get_url_builder("deb") + cls.repo_data = { + "name": "ubuntu", + "suite": "trusty", + "section": "main restricted", + "type": "deb", + "uri": "http://localhost/ubuntu" + } + + def test_get_repo_url(self): + self.assertEqual( + "http://localhost/ubuntu trusty main restricted", + self.builder.get_repo_url(self.repo_data) + ) + + +class TestYumUrlBuilder(base.TestCase): + @classmethod + def setUpClass(cls): + cls.builder = url_builder.get_url_builder("rpm") + cls.repo_data = { + "name": "centos", + "type": "rpm", + "uri": "http://localhost/os/x86_64" + } + + def test_get_repo_url(self): + self.assertEqual( + "http://localhost/os", + self.builder.get_repo_url(self.repo_data) + ) diff --git a/contrib/fuel_mirror/fuel_mirror/tests/test_utils.py b/contrib/fuel_mirror/fuel_mirror/tests/test_utils.py new file mode 100644 index 0000000..7024b50 --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/tests/test_utils.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import six + +from fuel_mirror.common import utils +from fuel_mirror.tests import base + + +class DictAsObj(object): + def __init__(self, d): + self.__dict__.update(d) + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + +class TestUtils(base.TestCase): + def test_lists_merge(self): + main = [{"a": 1, "b": 2, "c": 0}, {"a": 2, "b": 3, "c": 1}] + patch = [{"a": 2, "b": 4}, {"a": 3, "b": 5}] + utils.lists_merge( + main, + patch, + key="a" + ) + self.assertItemsEqual( + [{"a": 1, "b": 2, "c": 0}, + {"a": 2, "b": 4, "c": 1}, + {"a": 3, "b": 5}], + main + ) + + def test_first(self): + self.assertEqual( + 1, + utils.first(0, 1, 0), + ) + self.assertEqual( + 1, + utils.first(None, [], '', 1), + ) + self.assertIsNone( + utils.first(None, [], 0, ''), + ) + self.assertIsNone( + utils.first(), + ) + + def test_is_subdict(self): + self.assertFalse(utils.is_subdict({"c": 1}, {"a": 1, "b": 1})) + self.assertFalse(utils.is_subdict({"a": 1, "b": 2}, {"a": 1, "b": 1})) + self.assertFalse( + utils.is_subdict({"a": 1, "b": 1, "c": 2}, {"a": 1, "b": 1}) + ) + self.assertFalse( + utils.is_subdict({"a": 1, "b": None}, {"a": 1}) + ) + self.assertTrue(utils.is_subdict({}, {"a": 1})) + self.assertTrue(utils.is_subdict({"a": 1}, {"a": 1, "b": 1})) + self.assertTrue(utils.is_subdict({"a": 1, "b": 1}, {"a": 1, "b": 1})) + + @mock.patch("fuel_mirror.common.utils.open") + def test_get_fuel_settings(self, m_open): + m_open().__enter__.side_effect = [ + six.StringIO( + 'ADMIN_NETWORK:\n' + ' ipaddress: "10.20.0.4"\n' + 'FUEL_ACCESS:\n' + ' user: "test"\n' + ' password: "test_pwd"\n', + ), + OSError + ] + + self.assertEqual( + { + "server": "10.20.0.4", + "user": "test", + "password": "test_pwd", + }, + utils.get_fuel_settings() + ) + + self.assertEqual( + { + "server": "10.20.0.2", + "user": None, + "password": None, + }, + utils.get_fuel_settings() + ) diff --git a/contrib/fuel_mirror/fuel_mirror/tests/test_validate_config.py b/contrib/fuel_mirror/fuel_mirror/tests/test_validate_config.py new file mode 100644 index 0000000..331e057 --- /dev/null +++ b/contrib/fuel_mirror/fuel_mirror/tests/test_validate_config.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os.path +import yaml + +from fuel_mirror.tests import base + + +DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "data") + + +class TestValidateConfigs(base.TestCase): + def test_validate_data_files(self): + for f in os.listdir(DATA_DIR): + with open(os.path.join(DATA_DIR, f), "r") as fd: + data = yaml.load(fd) + # TODO(add input data validation scheme) + self.assertIn("groups", data) + self.assertIn("fuel_release_match", data) diff --git a/contrib/fuel_mirror/requirements.txt b/contrib/fuel_mirror/requirements.txt new file mode 100644 index 0000000..d07d40a --- /dev/null +++ b/contrib/fuel_mirror/requirements.txt @@ -0,0 +1,9 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +pbr>=0.8 +Babel>=1.3 +cliff>=1.7.0 +six>=1.5.2 +packetary diff --git a/contrib/fuel_mirror/scripts/fuel-createmirror b/contrib/fuel_mirror/scripts/fuel-createmirror new file mode 100755 index 0000000..c6471be --- /dev/null +++ b/contrib/fuel_mirror/scripts/fuel-createmirror @@ -0,0 +1,79 @@ +#!/bin/bash + +echo "This script is DEPRECATED. Please use fuel-mirror utility!" + +# This shell script was wraps the fuel-mirror utility to provide backward compatibility +# with previous version of tool. + +usage() { +cat <=0.10.0 + +coverage>=3.6 +discover +python-subunit>=0.0.18 +sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 +oslosphinx>=2.5.0 # Apache-2.0 +oslotest>=1.10.0 # Apache-2.0 +testrepository>=0.0.18 +testscenarios>=0.4 +testtools>=1.4.0 +cliff>=1.7.0 +six>=1.5.2 diff --git a/contrib/fuel_mirror/tox.ini b/contrib/fuel_mirror/tox.ini new file mode 100644 index 0000000..0a57b39 --- /dev/null +++ b/contrib/fuel_mirror/tox.ini @@ -0,0 +1,35 @@ +[tox] +minversion = 1.6 +envlist = py34,py27,py26,pep8 +skipsdist = True + +[testenv] +usedevelop = True +install_command = pip install -U {opts} {packages} +setenv = + VIRTUAL_ENV={envdir} +deps = -r{toxinidir}/test-requirements.txt +commands = python setup.py test --slowest --testr-args='{posargs:fuel_mirror}' + +[testenv:pep8] +commands = flake8 + +[testenv:venv] +commands = {posargs} + +[testenv:cover] +commands = python setup.py test --coverage --testr-args='{posargs:fuel_mirror}' + +[testenv:docs] +commands = python setup.py build_sphinx + +[testenv:debug] +commands = oslo_debug_helper {posargs} + +[flake8] +# E123, E125 skipped as they are invalid PEP-8. + +show-source = True +ignore = E123,E125 +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build diff --git a/packetary/drivers/deb_driver.py b/packetary/drivers/deb_driver.py index a21f24c..0c26f43 100644 --- a/packetary/drivers/deb_driver.py +++ b/packetary/drivers/deb_driver.py @@ -180,21 +180,19 @@ class DebRepositoryDriver(RepositoryDriverBase): def fork_repository(self, connection, repository, destination, source=False, locale=False): - """Overrides method of superclass.""" # TODO(download gpk) # TODO(sources and locales) - if not destination.endswith(os.path.sep): - destination += os.path.sep - - clone = copy.copy(repository) - clone.url = destination + new_repo = copy.copy(repository) + new_repo.url = utils.localize_repo_url(destination, repository.url) packages_file = utils.get_path_from_url( - self._get_url_of_metafile(clone, "Packages") + self._get_url_of_metafile(new_repo, "Packages") ) release_file = utils.get_path_from_url( - self._get_url_of_metafile(clone, "Release") + self._get_url_of_metafile(new_repo, "Release") + ) + self.logger.info( + "clone repository %s to %s", repository, new_repo.url ) - self.logger.info("clone repository %s to %s", repository, destination) utils.ensure_dir_exist(os.path.dirname(release_file)) release = deb822.Release() @@ -208,7 +206,7 @@ class DebRepositoryDriver(RepositoryDriverBase): open(packages_file, "ab").close() gzip.open(packages_file + ".gz", "ab").close() - return clone + return new_repo def _update_suite_index(self, repository): """Updates the Release file in the suite.""" diff --git a/packetary/drivers/rpm_driver.py b/packetary/drivers/rpm_driver.py index a00aef7..e5958b8 100644 --- a/packetary/drivers/rpm_driver.py +++ b/packetary/drivers/rpm_driver.py @@ -72,15 +72,10 @@ class RpmRepositoryDriver(RepositoryDriverBase): return (url.rstrip("/") for url in urls) def get_repository(self, connection, url, arch, consumer): - """Overrides method of superclass.""" - # Currently supported repositories, that has URL in following format: - # baseurl/{name}/{architecture} - # because the architecture is sentetic part of rpm repository URL - name = url.rsplit("/", 1)[-1] - baseurl = "/".join((url, arch, "")) + name = utils.get_path_from_url(url, False) consumer(Repository( name=name, - url=baseurl, + url=url + "/", architecture=arch, origin="" )) @@ -165,17 +160,14 @@ class RpmRepositoryDriver(RepositoryDriverBase): def fork_repository(self, connection, repository, destination, source=False, locale=False): - """Overrides method of superclass.""" # TODO(download gpk) # TODO(sources and locales) - destination = os.path.join( - destination, repository.name, - repository.architecture, "" - ) new_repo = copy.copy(repository) - new_repo.url = destination - self.logger.info("clone repository %s to %s", repository, destination) - utils.ensure_dir_exist(destination) + new_repo.url = utils.localize_repo_url(destination, repository.url) + self.logger.info( + "clone repository %s to %s", repository, new_repo.url + ) + utils.ensure_dir_exist(new_repo.url) self.rebuild_repository(new_repo, set()) return new_repo diff --git a/packetary/library/utils.py b/packetary/library/utils.py index a3df3c1..ca5599c 100644 --- a/packetary/library/utils.py +++ b/packetary/library/utils.py @@ -90,6 +90,16 @@ def get_path_from_url(url, ensure_file=True): return comps.path +def localize_repo_url(localurl, repo_url): + """Gets local repository url. + + :param localurl: the base local URL + :param repo_url: the origin URL of repository + :return: localurl + get_path_from_url(repo_url) + """ + return localurl.rstrip("/") + urlparse(repo_url).path + + def ensure_dir_exist(path): """Creates directory if it does not exist. diff --git a/packetary/tests/test_deb_driver.py b/packetary/tests/test_deb_driver.py index 48c2c25..3712c28 100644 --- a/packetary/tests/test_deb_driver.py +++ b/packetary/tests/test_deb_driver.py @@ -20,6 +20,7 @@ import six from packetary.drivers import deb_driver +from packetary.library.utils import localize_repo_url from packetary.tests import base from packetary.tests.stubs.generator import gen_package from packetary.tests.stubs.generator import gen_repository @@ -188,26 +189,29 @@ class TestDebDriver(base.TestCase): os.path.sep = "/" os.path.join = lambda *x: "/".join(x) utils.get_path_from_url = lambda x: x - repo = gen_repository(name=("trusty", "main"), url="http://localhost") + utils.localize_repo_url = localize_repo_url + repo = gen_repository( + name=("trusty", "main"), url="http://localhost/test/" + ) files = [ mock.MagicMock(), mock.MagicMock() ] open.side_effect = files - clone = self.driver.fork_repository(self.connection, repo, "/root") - self.assertEqual(repo.name, clone.name) - self.assertEqual(repo.architecture, clone.architecture) - self.assertEqual(repo.origin, clone.origin) - self.assertEqual("/root/", clone.url) + new_repo = self.driver.fork_repository(self.connection, repo, "/root") + self.assertEqual(repo.name, new_repo.name) + self.assertEqual(repo.architecture, new_repo.architecture) + self.assertEqual(repo.origin, new_repo.origin) + self.assertEqual("/root/test/", new_repo.url) utils.ensure_dir_exist.assert_called_once_with(os.path.dirname()) open.assert_any_call( - "/root/dists/trusty/main/binary-amd64/Release", "wb" + "/root/test/dists/trusty/main/binary-amd64/Release", "wb" ) open.assert_any_call( - "/root/dists/trusty/main/binary-amd64/Packages", "ab" + "/root/test/dists/trusty/main/binary-amd64/Packages", "ab" ) gzip.open.assert_called_once_with( - "/root/dists/trusty/main/binary-amd64/Packages.gz", "ab" + "/root/test/dists/trusty/main/binary-amd64/Packages.gz", "ab" ) @mock.patch.multiple( diff --git a/packetary/tests/test_rpm_driver.py b/packetary/tests/test_rpm_driver.py index a4caec4..1fb001e 100644 --- a/packetary/tests/test_rpm_driver.py +++ b/packetary/tests/test_rpm_driver.py @@ -20,6 +20,7 @@ import sys import six +from packetary.library.utils import localize_repo_url from packetary.objects import FileChecksum from packetary.tests import base from packetary.tests.stubs.generator import gen_repository @@ -65,12 +66,15 @@ class TestRpmDriver(base.TestCase): repos = [] self.driver.get_repository( - self.connection, "http://host/centos/os", "x86_64", repos.append + self.connection, + "http://host/centos/os/x86_64", + "x86_64", + repos.append ) self.assertEqual(1, len(repos)) repo = repos[0] - self.assertEqual("os", repo.name) + self.assertEqual("/centos/os/x86_64", repo.name) self.assertEqual("", repo.origin) self.assertEqual("x86_64", repo.architecture) self.assertEqual("http://host/centos/os/x86_64/", repo.url) @@ -189,16 +193,17 @@ class TestRpmDriver(base.TestCase): @mock.patch("packetary.drivers.rpm_driver.utils") def test_fork_repository(self, utils): - repo = gen_repository("os", url="http://localhost/os/x86_64") - clone = self.driver.fork_repository( + repo = gen_repository("os", url="http://localhost/os/x86_64/") + utils.localize_repo_url = localize_repo_url + new_repo = self.driver.fork_repository( self.connection, repo, "/repo" ) utils.ensure_dir_exist.assert_called_once_with("/repo/os/x86_64/") - self.assertEqual(repo.name, clone.name) - self.assertEqual(repo.architecture, clone.architecture) - self.assertEqual("/repo/os/x86_64/", clone.url) + self.assertEqual(repo.name, new_repo.name) + self.assertEqual(repo.architecture, new_repo.architecture) + self.assertEqual("/repo/os/x86_64/", new_repo.url) self.createrepo.MetaDataGenerator()\ .doFinalMove.assert_called_once_with()