Change repositories list in config

Allow use to add arbitrary repository to the list.

Change-Id: I7895a9a66d4499aabcd5eb9bd037e5b2bc1fa83c

doc/source/config/index.rst

@@ -72,3 +72,12 @@ override values from include in following documents::
   - override_basic_value
   ---
   override_value: from_include
+
+Configuration file sections
+===========================
+
+Here you can find description of configuration parameters in these sections:
+
+.. toctree::
+
+  repositories

doc/source/config/repositories.rst

@@ -0,0 +1,60 @@
+.. _config_repositories:
+
+========================
+``repositories`` section
+========================
+
+This section contains information about repositories with component definitions
+that should be cloned by :command:`ccp fetch` command or used by other
+:command:`ccp` commands.
+
+Section-level parameters
+========================
+
+.. describe:: clone
+
+  Run :command:`ccp fetch` analogue before running other commands. Default:
+  ``true``
+
+.. describe:: clone_concurrency
+
+  Number of threads to use while cloning repos. Defaults to number of CPU cores
+  available.
+
+.. describe:: repos
+
+  List of repository definitions (see :ref:`below <config_repo_def>`) that
+  should be used by CCP tool. Defaults to a list of repos provided by CCP
+  upstream.
+
+.. _config_repo_path:
+
+.. describe:: path
+
+  Path to a dir where all repos are to be cloned or should be expected to be
+  present.
+
+.. describe:: skip_empty
+
+  Ignore empty repositories. Default: ``true``
+
+.. _config_repo_def:
+
+Repository definitions
+======================
+
+Every item from this list describes one component repository that should be
+downloaded or used by CCP tool.
+
+.. describe:: name
+
+  The name of the component, this is used as a name of directory in
+  :ref:`path <config_repo_path>` to clone or find component repo.
+
+.. describe:: git_url
+
+  The URL where repo should be cloned from
+
+.. describe:: git_ref
+
+  Git ref that should be checked out

doc/source/index.rst

@@ -14,7 +14,7 @@ User docs
 
    quickstart
    monitoring_and_logging
-   config
+   config/index
 
 Advanced topics
 ---------------

fuel_ccp/build.py

@@ -365,9 +365,9 @@ def build_components(components=None):
     dockerfiles = {}
     try:
         match = not bool(components)
-        for repository_name in CONF.repositories.names:
+        for repository_def in CONF.repositories.repos:
             dockerfiles.update(
-                find_dockerfiles(repository_name, match=match))
+                find_dockerfiles(repository_def['name'], match=match))
 
         find_dependencies(dockerfiles)
 

fuel_ccp/cli.py

@@ -86,7 +86,7 @@ class Deploy(BaseCommand):
 
 
 def do_fetch():
-    fetch.fetch_repositories(CONF.repositories.names)
+    fetch.fetch_repositories()
 
 
 class Fetch(BaseCommand):

fuel_ccp/common/utils.py

@@ -25,7 +25,7 @@ def get_resource_path(path):
 
 
 def get_config_paths():
-    components = list(CONF.repositories.names)
+    components = [d['name'] for d in CONF.repositories.repos]
     paths = []
     # Order does matter. At first we add global defaults.
     for conf_path in ("resources/defaults.yaml", "resources/globals.yaml"):
@@ -48,13 +48,13 @@ def get_deploy_components_info(rendering_context=None):
         rendering_context = CONF.configs._dict
     components_map = {}
 
-    for component in CONF.repositories.names:
+    for component_ref in CONF.repositories.repos:
+        component_name = component_ref['name']
         service_dir = os.path.join(CONF.repositories.path,
-                                   component,
+                                   component_name,
                                    'service')
         if not os.path.isdir(service_dir):
             continue
-        component_name = component
         REPO_NAME_PREFIX = "fuel-ccp-"
         if component_name.startswith(REPO_NAME_PREFIX):
             component_name = component_name[len(REPO_NAME_PREFIX):]

fuel_ccp/config/repositories.py

@@ -29,12 +29,10 @@ DEFAULTS = {
         'clone_concurrency': multiprocessing.cpu_count(),
         'skip_empty': True,
         'path': os.path.expanduser('~/ccp-repos/'),
-        'hostname': 'git.openstack.org',
+        'repos': [{
-        'port': 443,
+            'name': name,
-        'protocol': 'https',
+            'git_url': 'https://git.openstack.org/openstack/{}'.format(name),
-        'project': 'openstack',
+        } for name in DEFAULT_REPOS],
-        'username': None,
-        'names': DEFAULT_REPOS,
     },
 }
 
@@ -47,18 +45,19 @@ SCHEMA = {
             'clone_concurrency': {'type': 'integer'},
             'skip_empty': {'type': 'boolean'},
             'path': {'type': 'string'},
-            'hostname': {'type': 'string'},
+            'repos': {
-            'port': {'type': 'integer'},
+                'type': 'array',
-            'protocol': {'type': 'string'},
+                'items': {
-            'project': {'type': 'string'},
+                    'type': 'object',
-            'username': {'anyOf': [{'type': 'string'}, {'type': 'null'}]},
+                    'additionalProperties': False,
-            'names': {'type': 'array', 'items': {'type': 'string'}},
+                    'required': ['name', 'git_url'],
+                    'properties': {
+                        'name': {'type': 'string'},
+                        'git_url': {'type': 'string'},
+                        'git_ref': {'type': 'string'},
+                    },
+                },
+            },
         },
     },
 }
-
-for repo in DEFAULT_REPOS:
-    conf_name = repo.replace('-', '_')
-    SCHEMA['repositories']['properties'][conf_name] = \
-        {'anyOf': [{'type': 'string'}, {'type': 'null'}]}
-    DEFAULTS['repositories'][conf_name] = None

fuel_ccp/fetch.py

@@ -14,28 +14,26 @@ LOG = logging.getLogger(__name__)
 FETCH_TIMEOUT = 2 ** 16  # in seconds
 
 
-def fetch_repository(repository_name):
+def fetch_repository(repository_def):
-    dest_dir = os.path.join(CONF.repositories.path, repository_name)
+    name = repository_def['name']
+    dest_dir = os.path.join(CONF.repositories.path, name)
     if os.path.isdir(dest_dir):
-        LOG.debug('%s was already cloned, skipping', repository_name)
+        LOG.debug('%s was already cloned, skipping', name)
         return
-    git_url = getattr(CONF.repositories, repository_name.replace('-', '_'))
+    git_url = repository_def['git_url']
-    if git_url is None:
+    git_ref = repository_def.get('git_ref')
-        username = CONF.repositories.username
+    if git_ref:
-        if username is None:
+        kwargs = {'branch': git_ref}
-            username = ''
+    else:
-        else:
+        kwargs = {}
-            username = username + '@'
+    LOG.debug('Clonning %s from %s to %s', name, git_url, dest_dir)
-        fmt = '{0.protocol}://{1}{0.hostname}:{0.port}/{0.project}/{2}'
+    git.Repo.clone_from(git_url, dest_dir, **kwargs)
-        git_url = fmt.format(CONF.repositories, username, repository_name)
+    LOG.info('Cloned %s repo', name)
-    LOG.debug('Clonning %s from %s', repository_name, git_url)
-    git.Repo.clone_from(git_url, dest_dir)
-    LOG.info('Cloned %s repo', repository_name)
 
 
-def fetch_repositories(repository_names=None):
+def fetch_repositories(repository_defs=None):
-    if repository_names is None:
+    if repository_defs is None:
-        repository_names = CONF.repositories.names
+        repository_defs = CONF.repositories.repos
 
     LOG.info('Cloning repositories into %s', CONF.repositories.path)
 
@@ -44,9 +42,9 @@ def fetch_repositories(repository_names=None):
             max_workers=CONF.repositories.clone_concurrency) as executor:
         future_list = []
         try:
-            for repository_name in repository_names:
+            for repository_def in repository_defs:
                 future_list.append(executor.submit(
-                    fetch_repository, repository_name
+                    fetch_repository, repository_def
                 ))
 
             for future in future_list:

fuel_ccp/tests/common/test_utils.py

@@ -36,7 +36,7 @@ class TestUtils(base.TestCase):
         base_dir = os.path.dirname(__file__)
 
         self.conf.repositories.path = os.path.join(base_dir, "test_repo_dir")
-        self.conf.repositories.names = ["component"]
+        self.conf.repositories.repos = [{"name": "component"}]
 
         res = (
             utils.get_deploy_components_info()["keystone"]["service_content"]
@@ -73,7 +73,7 @@ class TestUtils(base.TestCase):
         base_dir = os.path.dirname(__file__)
 
         self.conf.repositories.path = os.path.join(base_dir, "test_repo_dir")
-        self.conf.repositories.names = ["component"]
+        self.conf.repositories.repos = [{"name": "component"}]
 
         config.load_component_defaults()
 
@@ -111,7 +111,7 @@ class TestUtils(base.TestCase):
         base_dir = os.path.dirname(__file__)
 
         self.conf.repositories.path = os.path.join(base_dir, "test_repo_dir")
-        self.conf.repositories.names = ["component"]
+        self.conf.repositories.repos = [{"name": "component"}]
 
         config.load_component_defaults()
 

fuel_ccp/tests/test_cli.py

@@ -27,7 +27,7 @@ class SafeCCPApp(cli.CCPApp):
         # Debug does magic in cliff, we need it always on
         parser = super(SafeCCPApp, self).build_option_parser(
             description, version, argparse_kwargs)
-        parser.set_defaults(debug=True)
+        parser.set_defaults(debug=True, verbosity_level=2)
         return parser
 
     def get_fuzzy_matches(self, cmd):
@@ -143,7 +143,7 @@ class TestFetch(TestParser):
 
     def test_parser(self):
         self._run_app()
-        self.fetch_mock.assert_called_once_with(config.CONF.repositories.names)
+        self.fetch_mock.assert_called_once_with()
 
 
 class TestCleanup(TestParser):

fuel_ccp/tests/test_fetch.py

@@ -3,92 +3,45 @@ import os
 import fixtures
 from fuel_ccp import fetch
 from fuel_ccp.tests import base
-import mock
 import testscenarios
 
 
-@mock.patch('git.Repo.clone_from')
 class TestFetch(testscenarios.WithScenarios, base.TestCase):
+    component_def = {'name': 'compname', 'git_url': 'theurl'}
+    update_def = {}
+    expected_clone_call = None
+    dir_exists = False
+
     scenarios = [
-        ("default", {
+        ('exists', {'dir_exists': True}),
-            "option": None,
-            "value": None,
-            "url": "https://git.openstack.org:443/openstack/%s"}),
-        ("hostname", {
-            "option": "hostname",
-            "value": "host.name",
-            "url": "https://host.name:443/openstack/%s"}),
-        ('username', {
-            "option": "username",
-            "value": "someuser",
-            "url": "https://someuser@git.openstack.org:443/openstack/%s",
-        }),
-        ('port', {
-            "option": "port",
-            "value": "9999",
-            'url': "https://git.openstack.org:9999/openstack/%s",
-        }),
-        ('protocol', {
-            "option": "protocol",
-            "value": "ssh",
-            'url': "ssh://git.openstack.org:443/openstack/%s",
-        }),
-        ('protocol', {
-            "option": "protocol",
-            "value": "http",
-            'url': "http://git.openstack.org:443/openstack/%s",
-        }),
-        ('protocol', {
-            "option": "protocol",
-            "value": "git",
-            'url': "git://git.openstack.org:443/openstack/%s",
-        }),
-        ('protocol', {
-            "option": "protocol",
-            "value": "https",
-            'url': "https://git.openstack.org:443/openstack/%s",
-        }),
-        ('project', {
-            "option": "project",
-            "value": "someproject",
-            'url': "https://git.openstack.org:443/someproject/%s",
-        })
     ]
-    url = None
-    option = None
-    value = None
 
     def setUp(self):
         super(TestFetch, self).setUp()
         # Creating temporaty directory for repos
-        tmp_dir = fixtures.TempDir()
+        self.tmp_path = self.useFixture(fixtures.TempDir()).path
-        tmp_dir.setUp()
-        self.tmp_path = tmp_dir.path
         self.conf['repositories']['path'] = self.tmp_path
-        # Create temporary directory for openstack-base to not clone it
+        fixture = fixtures.MockPatch('git.Repo.clone_from')
-        os.mkdir(os.path.join(self.tmp_path, 'ms-openstack-base'))
+        self.mock_clone = self.useFixture(fixture).mock
 
-    def test_fetch_default_repositories(self, m_clone):
+    def test_fetch_repository(self):
-        if self.option is not None:
+        component_def = self.component_def.copy()
-            self.conf['repositories'][self.option] = self.value
+        component_def.update(self.update_def)
-        self.conf['repositories']['path'] = self.tmp_path
+
-        components = ['fuel-ccp-debian-base',
+        fixture = fixtures.MockPatch('os.path.isdir')
-                      'fuel-ccp-entrypoint',
+        isdir_mock = self.useFixture(fixture).mock
-                      'fuel-ccp-etcd',
+        isdir_mock.return_value = self.dir_exists
-                      'fuel-ccp-glance',
+
-                      'fuel-ccp-horizon',
+        fetch.fetch_repository(component_def)
-                      'fuel-ccp-keystone',
+
-                      'fuel-ccp-mariadb',
+        git_path = os.path.join(self.tmp_path, component_def['name'])
-                      'fuel-ccp-memcached',
+        isdir_mock.assert_called_once_with(git_path)
-                      'fuel-ccp-neutron',
+        if self.expected_clone_call:
-                      'fuel-ccp-nova',
+            git_ref = component_def.get('git_ref')
-                      'fuel-ccp-rabbitmq',
+            if git_ref:
-                      'fuel-ccp-stacklight']
+                self.mock_clone.assert_called_once_with(
-        expected_calls = [
+                    'theurl', git_path, branch=git_ref)
-            mock.call(
+            else:
-                self.url % (component),
+                self.mock_clone.assert_called_once_with('theurl', git_path)
-                os.path.join(self.tmp_path, component))
+        else:
-            for component in components]
+            self.mock_clone.assert_not_called()
-        for component, expected_call in zip(components, expected_calls):
-            fetch.fetch_repository(component)
-            self.assertIn(expected_call, m_clone.call_args_list)