From 855fda0ef3f2a07f25d28164cc0ee305f4b8c2d9 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Tue, 11 Oct 2016 16:28:51 +0300 Subject: [PATCH] Change repositories list in config Allow use to add arbitrary repository to the list. Change-Id: I7895a9a66d4499aabcd5eb9bd037e5b2bc1fa83c --- doc/source/{config.rst => config/index.rst} | 9 ++ doc/source/config/repositories.rst | 60 +++++++++++ doc/source/index.rst | 2 +- fuel_ccp/build.py | 4 +- fuel_ccp/cli.py | 2 +- fuel_ccp/common/utils.py | 8 +- fuel_ccp/config/repositories.py | 35 ++++--- fuel_ccp/fetch.py | 38 ++++--- fuel_ccp/tests/common/test_utils.py | 6 +- fuel_ccp/tests/test_cli.py | 4 +- fuel_ccp/tests/test_fetch.py | 107 ++++++-------------- 11 files changed, 147 insertions(+), 128 deletions(-) rename doc/source/{config.rst => config/index.rst} (91%) create mode 100644 doc/source/config/repositories.rst diff --git a/doc/source/config.rst b/doc/source/config/index.rst similarity index 91% rename from doc/source/config.rst rename to doc/source/config/index.rst index 9fcb4bd5..3997ec73 100644 --- a/doc/source/config.rst +++ b/doc/source/config/index.rst @@ -72,3 +72,12 @@ override values from include in following documents:: - override_basic_value --- override_value: from_include + +Configuration file sections +=========================== + +Here you can find description of configuration parameters in these sections: + +.. toctree:: + + repositories diff --git a/doc/source/config/repositories.rst b/doc/source/config/repositories.rst new file mode 100644 index 00000000..417af6a2 --- /dev/null +++ b/doc/source/config/repositories.rst @@ -0,0 +1,60 @@ +.. _config_repositories: + +======================== +``repositories`` section +======================== + +This section contains information about repositories with component definitions +that should be cloned by :command:`ccp fetch` command or used by other +:command:`ccp` commands. + +Section-level parameters +======================== + +.. describe:: clone + + Run :command:`ccp fetch` analogue before running other commands. Default: + ``true`` + +.. describe:: clone_concurrency + + Number of threads to use while cloning repos. Defaults to number of CPU cores + available. + +.. describe:: repos + + List of repository definitions (see :ref:`below `) that + should be used by CCP tool. Defaults to a list of repos provided by CCP + upstream. + +.. _config_repo_path: + +.. describe:: path + + Path to a dir where all repos are to be cloned or should be expected to be + present. + +.. describe:: skip_empty + + Ignore empty repositories. Default: ``true`` + +.. _config_repo_def: + +Repository definitions +====================== + +Every item from this list describes one component repository that should be +downloaded or used by CCP tool. + +.. describe:: name + + The name of the component, this is used as a name of directory in + :ref:`path ` to clone or find component repo. + +.. describe:: git_url + + The URL where repo should be cloned from + +.. describe:: git_ref + + Git ref that should be checked out diff --git a/doc/source/index.rst b/doc/source/index.rst index 5e21ed09..04528b20 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -14,7 +14,7 @@ User docs quickstart monitoring_and_logging - config + config/index Advanced topics --------------- diff --git a/fuel_ccp/build.py b/fuel_ccp/build.py index 2e8d3ea5..5926b602 100644 --- a/fuel_ccp/build.py +++ b/fuel_ccp/build.py @@ -365,9 +365,9 @@ def build_components(components=None): dockerfiles = {} try: match = not bool(components) - for repository_name in CONF.repositories.names: + for repository_def in CONF.repositories.repos: dockerfiles.update( - find_dockerfiles(repository_name, match=match)) + find_dockerfiles(repository_def['name'], match=match)) find_dependencies(dockerfiles) diff --git a/fuel_ccp/cli.py b/fuel_ccp/cli.py index fd3d3ec7..69580fd5 100644 --- a/fuel_ccp/cli.py +++ b/fuel_ccp/cli.py @@ -86,7 +86,7 @@ class Deploy(BaseCommand): def do_fetch(): - fetch.fetch_repositories(CONF.repositories.names) + fetch.fetch_repositories() class Fetch(BaseCommand): diff --git a/fuel_ccp/common/utils.py b/fuel_ccp/common/utils.py index eb3a6d8f..da674b32 100644 --- a/fuel_ccp/common/utils.py +++ b/fuel_ccp/common/utils.py @@ -25,7 +25,7 @@ def get_resource_path(path): def get_config_paths(): - components = list(CONF.repositories.names) + components = [d['name'] for d in CONF.repositories.repos] paths = [] # Order does matter. At first we add global defaults. for conf_path in ("resources/defaults.yaml", "resources/globals.yaml"): @@ -48,13 +48,13 @@ def get_deploy_components_info(rendering_context=None): rendering_context = CONF.configs._dict components_map = {} - for component in CONF.repositories.names: + for component_ref in CONF.repositories.repos: + component_name = component_ref['name'] service_dir = os.path.join(CONF.repositories.path, - component, + component_name, 'service') if not os.path.isdir(service_dir): continue - component_name = component REPO_NAME_PREFIX = "fuel-ccp-" if component_name.startswith(REPO_NAME_PREFIX): component_name = component_name[len(REPO_NAME_PREFIX):] diff --git a/fuel_ccp/config/repositories.py b/fuel_ccp/config/repositories.py index 542ff152..cf40393b 100644 --- a/fuel_ccp/config/repositories.py +++ b/fuel_ccp/config/repositories.py @@ -29,12 +29,10 @@ DEFAULTS = { 'clone_concurrency': multiprocessing.cpu_count(), 'skip_empty': True, 'path': os.path.expanduser('~/ccp-repos/'), - 'hostname': 'git.openstack.org', - 'port': 443, - 'protocol': 'https', - 'project': 'openstack', - 'username': None, - 'names': DEFAULT_REPOS, + 'repos': [{ + 'name': name, + 'git_url': 'https://git.openstack.org/openstack/{}'.format(name), + } for name in DEFAULT_REPOS], }, } @@ -47,18 +45,19 @@ SCHEMA = { 'clone_concurrency': {'type': 'integer'}, 'skip_empty': {'type': 'boolean'}, 'path': {'type': 'string'}, - 'hostname': {'type': 'string'}, - 'port': {'type': 'integer'}, - 'protocol': {'type': 'string'}, - 'project': {'type': 'string'}, - 'username': {'anyOf': [{'type': 'string'}, {'type': 'null'}]}, - 'names': {'type': 'array', 'items': {'type': 'string'}}, + 'repos': { + 'type': 'array', + 'items': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['name', 'git_url'], + 'properties': { + 'name': {'type': 'string'}, + 'git_url': {'type': 'string'}, + 'git_ref': {'type': 'string'}, + }, + }, + }, }, }, } - -for repo in DEFAULT_REPOS: - conf_name = repo.replace('-', '_') - SCHEMA['repositories']['properties'][conf_name] = \ - {'anyOf': [{'type': 'string'}, {'type': 'null'}]} - DEFAULTS['repositories'][conf_name] = None diff --git a/fuel_ccp/fetch.py b/fuel_ccp/fetch.py index c24a7c9e..459660ba 100644 --- a/fuel_ccp/fetch.py +++ b/fuel_ccp/fetch.py @@ -14,28 +14,26 @@ LOG = logging.getLogger(__name__) FETCH_TIMEOUT = 2 ** 16 # in seconds -def fetch_repository(repository_name): - dest_dir = os.path.join(CONF.repositories.path, repository_name) +def fetch_repository(repository_def): + name = repository_def['name'] + dest_dir = os.path.join(CONF.repositories.path, name) if os.path.isdir(dest_dir): - LOG.debug('%s was already cloned, skipping', repository_name) + LOG.debug('%s was already cloned, skipping', name) return - git_url = getattr(CONF.repositories, repository_name.replace('-', '_')) - if git_url is None: - username = CONF.repositories.username - if username is None: - username = '' - else: - username = username + '@' - fmt = '{0.protocol}://{1}{0.hostname}:{0.port}/{0.project}/{2}' - git_url = fmt.format(CONF.repositories, username, repository_name) - LOG.debug('Clonning %s from %s', repository_name, git_url) - git.Repo.clone_from(git_url, dest_dir) - LOG.info('Cloned %s repo', repository_name) + git_url = repository_def['git_url'] + git_ref = repository_def.get('git_ref') + if git_ref: + kwargs = {'branch': git_ref} + else: + kwargs = {} + LOG.debug('Clonning %s from %s to %s', name, git_url, dest_dir) + git.Repo.clone_from(git_url, dest_dir, **kwargs) + LOG.info('Cloned %s repo', name) -def fetch_repositories(repository_names=None): - if repository_names is None: - repository_names = CONF.repositories.names +def fetch_repositories(repository_defs=None): + if repository_defs is None: + repository_defs = CONF.repositories.repos LOG.info('Cloning repositories into %s', CONF.repositories.path) @@ -44,9 +42,9 @@ def fetch_repositories(repository_names=None): max_workers=CONF.repositories.clone_concurrency) as executor: future_list = [] try: - for repository_name in repository_names: + for repository_def in repository_defs: future_list.append(executor.submit( - fetch_repository, repository_name + fetch_repository, repository_def )) for future in future_list: diff --git a/fuel_ccp/tests/common/test_utils.py b/fuel_ccp/tests/common/test_utils.py index b660b8be..e39780c0 100644 --- a/fuel_ccp/tests/common/test_utils.py +++ b/fuel_ccp/tests/common/test_utils.py @@ -36,7 +36,7 @@ class TestUtils(base.TestCase): base_dir = os.path.dirname(__file__) self.conf.repositories.path = os.path.join(base_dir, "test_repo_dir") - self.conf.repositories.names = ["component"] + self.conf.repositories.repos = [{"name": "component"}] res = ( utils.get_deploy_components_info()["keystone"]["service_content"] @@ -73,7 +73,7 @@ class TestUtils(base.TestCase): base_dir = os.path.dirname(__file__) self.conf.repositories.path = os.path.join(base_dir, "test_repo_dir") - self.conf.repositories.names = ["component"] + self.conf.repositories.repos = [{"name": "component"}] config.load_component_defaults() @@ -111,7 +111,7 @@ class TestUtils(base.TestCase): base_dir = os.path.dirname(__file__) self.conf.repositories.path = os.path.join(base_dir, "test_repo_dir") - self.conf.repositories.names = ["component"] + self.conf.repositories.repos = [{"name": "component"}] config.load_component_defaults() diff --git a/fuel_ccp/tests/test_cli.py b/fuel_ccp/tests/test_cli.py index 0f9d8f68..aea43cf2 100644 --- a/fuel_ccp/tests/test_cli.py +++ b/fuel_ccp/tests/test_cli.py @@ -27,7 +27,7 @@ class SafeCCPApp(cli.CCPApp): # Debug does magic in cliff, we need it always on parser = super(SafeCCPApp, self).build_option_parser( description, version, argparse_kwargs) - parser.set_defaults(debug=True) + parser.set_defaults(debug=True, verbosity_level=2) return parser def get_fuzzy_matches(self, cmd): @@ -143,7 +143,7 @@ class TestFetch(TestParser): def test_parser(self): self._run_app() - self.fetch_mock.assert_called_once_with(config.CONF.repositories.names) + self.fetch_mock.assert_called_once_with() class TestCleanup(TestParser): diff --git a/fuel_ccp/tests/test_fetch.py b/fuel_ccp/tests/test_fetch.py index 918fd5a6..49db73c5 100644 --- a/fuel_ccp/tests/test_fetch.py +++ b/fuel_ccp/tests/test_fetch.py @@ -3,92 +3,45 @@ import os import fixtures from fuel_ccp import fetch from fuel_ccp.tests import base -import mock import testscenarios -@mock.patch('git.Repo.clone_from') class TestFetch(testscenarios.WithScenarios, base.TestCase): + component_def = {'name': 'compname', 'git_url': 'theurl'} + update_def = {} + expected_clone_call = None + dir_exists = False + scenarios = [ - ("default", { - "option": None, - "value": None, - "url": "https://git.openstack.org:443/openstack/%s"}), - ("hostname", { - "option": "hostname", - "value": "host.name", - "url": "https://host.name:443/openstack/%s"}), - ('username', { - "option": "username", - "value": "someuser", - "url": "https://someuser@git.openstack.org:443/openstack/%s", - }), - ('port', { - "option": "port", - "value": "9999", - 'url': "https://git.openstack.org:9999/openstack/%s", - }), - ('protocol', { - "option": "protocol", - "value": "ssh", - 'url': "ssh://git.openstack.org:443/openstack/%s", - }), - ('protocol', { - "option": "protocol", - "value": "http", - 'url': "http://git.openstack.org:443/openstack/%s", - }), - ('protocol', { - "option": "protocol", - "value": "git", - 'url': "git://git.openstack.org:443/openstack/%s", - }), - ('protocol', { - "option": "protocol", - "value": "https", - 'url': "https://git.openstack.org:443/openstack/%s", - }), - ('project', { - "option": "project", - "value": "someproject", - 'url': "https://git.openstack.org:443/someproject/%s", - }) + ('exists', {'dir_exists': True}), ] - url = None - option = None - value = None def setUp(self): super(TestFetch, self).setUp() # Creating temporaty directory for repos - tmp_dir = fixtures.TempDir() - tmp_dir.setUp() - self.tmp_path = tmp_dir.path + self.tmp_path = self.useFixture(fixtures.TempDir()).path self.conf['repositories']['path'] = self.tmp_path - # Create temporary directory for openstack-base to not clone it - os.mkdir(os.path.join(self.tmp_path, 'ms-openstack-base')) + fixture = fixtures.MockPatch('git.Repo.clone_from') + self.mock_clone = self.useFixture(fixture).mock - def test_fetch_default_repositories(self, m_clone): - if self.option is not None: - self.conf['repositories'][self.option] = self.value - self.conf['repositories']['path'] = self.tmp_path - components = ['fuel-ccp-debian-base', - 'fuel-ccp-entrypoint', - 'fuel-ccp-etcd', - 'fuel-ccp-glance', - 'fuel-ccp-horizon', - 'fuel-ccp-keystone', - 'fuel-ccp-mariadb', - 'fuel-ccp-memcached', - 'fuel-ccp-neutron', - 'fuel-ccp-nova', - 'fuel-ccp-rabbitmq', - 'fuel-ccp-stacklight'] - expected_calls = [ - mock.call( - self.url % (component), - os.path.join(self.tmp_path, component)) - for component in components] - for component, expected_call in zip(components, expected_calls): - fetch.fetch_repository(component) - self.assertIn(expected_call, m_clone.call_args_list) + def test_fetch_repository(self): + component_def = self.component_def.copy() + component_def.update(self.update_def) + + fixture = fixtures.MockPatch('os.path.isdir') + isdir_mock = self.useFixture(fixture).mock + isdir_mock.return_value = self.dir_exists + + fetch.fetch_repository(component_def) + + git_path = os.path.join(self.tmp_path, component_def['name']) + isdir_mock.assert_called_once_with(git_path) + if self.expected_clone_call: + git_ref = component_def.get('git_ref') + if git_ref: + self.mock_clone.assert_called_once_with( + 'theurl', git_path, branch=git_ref) + else: + self.mock_clone.assert_called_once_with('theurl', git_path) + else: + self.mock_clone.assert_not_called()