Change repositories list in config
Allow use to add arbitrary repository to the list. Change-Id: I7895a9a66d4499aabcd5eb9bd037e5b2bc1fa83c
This commit is contained in:
Yuriy Taraday
parent
cb6bada442
commit
855fda0ef3
|
|
@ -72,3 +72,12 @@ override values from include in following documents::
|
|||
- override_basic_value
|
||||
---
|
||||
override_value: from_include
|
||||
|
||||
Configuration file sections
|
||||
===========================
|
||||
|
||||
Here you can find description of configuration parameters in these sections:
|
||||
|
||||
.. toctree::
|
||||
|
||||
repositories
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
.. _config_repositories:
|
||||
|
||||
========================
|
||||
``repositories`` section
|
||||
========================
|
||||
|
||||
This section contains information about repositories with component definitions
|
||||
that should be cloned by :command:`ccp fetch` command or used by other
|
||||
:command:`ccp` commands.
|
||||
|
||||
Section-level parameters
|
||||
========================
|
||||
|
||||
.. describe:: clone
|
||||
|
||||
Run :command:`ccp fetch` analogue before running other commands. Default:
|
||||
``true``
|
||||
|
||||
.. describe:: clone_concurrency
|
||||
|
||||
Number of threads to use while cloning repos. Defaults to number of CPU cores
|
||||
available.
|
||||
|
||||
.. describe:: repos
|
||||
|
||||
List of repository definitions (see :ref:`below <config_repo_def>`) that
|
||||
should be used by CCP tool. Defaults to a list of repos provided by CCP
|
||||
upstream.
|
||||
|
||||
.. _config_repo_path:
|
||||
|
||||
.. describe:: path
|
||||
|
||||
Path to a dir where all repos are to be cloned or should be expected to be
|
||||
present.
|
||||
|
||||
.. describe:: skip_empty
|
||||
|
||||
Ignore empty repositories. Default: ``true``
|
||||
|
||||
.. _config_repo_def:
|
||||
|
||||
Repository definitions
|
||||
======================
|
||||
|
||||
Every item from this list describes one component repository that should be
|
||||
downloaded or used by CCP tool.
|
||||
|
||||
.. describe:: name
|
||||
|
||||
The name of the component, this is used as a name of directory in
|
||||
:ref:`path <config_repo_path>` to clone or find component repo.
|
||||
|
||||
.. describe:: git_url
|
||||
|
||||
The URL where repo should be cloned from
|
||||
|
||||
.. describe:: git_ref
|
||||
|
||||
Git ref that should be checked out
|
||||
|
|
@ -14,7 +14,7 @@ User docs
|
|||
|
||||
quickstart
|
||||
monitoring_and_logging
|
||||
config
|
||||
config/index
|
||||
|
||||
Advanced topics
|
||||
---------------
|
||||
|
|
|
|||
|
|
@ -365,9 +365,9 @@ def build_components(components=None):
|
|||
dockerfiles = {}
|
||||
try:
|
||||
match = not bool(components)
|
||||
for repository_name in CONF.repositories.names:
|
||||
for repository_def in CONF.repositories.repos:
|
||||
dockerfiles.update(
|
||||
find_dockerfiles(repository_name, match=match))
|
||||
find_dockerfiles(repository_def['name'], match=match))
|
||||
|
||||
find_dependencies(dockerfiles)
|
||||
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ class Deploy(BaseCommand):
|
|||
|
||||
|
||||
def do_fetch():
|
||||
fetch.fetch_repositories(CONF.repositories.names)
|
||||
fetch.fetch_repositories()
|
||||
|
||||
|
||||
class Fetch(BaseCommand):
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ def get_resource_path(path):
|
|||
|
||||
|
||||
def get_config_paths():
|
||||
components = list(CONF.repositories.names)
|
||||
components = [d['name'] for d in CONF.repositories.repos]
|
||||
paths = []
|
||||
# Order does matter. At first we add global defaults.
|
||||
for conf_path in ("resources/defaults.yaml", "resources/globals.yaml"):
|
||||
|
|
@ -48,13 +48,13 @@ def get_deploy_components_info(rendering_context=None):
|
|||
rendering_context = CONF.configs._dict
|
||||
components_map = {}
|
||||
|
||||
for component in CONF.repositories.names:
|
||||
for component_ref in CONF.repositories.repos:
|
||||
component_name = component_ref['name']
|
||||
service_dir = os.path.join(CONF.repositories.path,
|
||||
component,
|
||||
component_name,
|
||||
'service')
|
||||
if not os.path.isdir(service_dir):
|
||||
continue
|
||||
component_name = component
|
||||
REPO_NAME_PREFIX = "fuel-ccp-"
|
||||
if component_name.startswith(REPO_NAME_PREFIX):
|
||||
component_name = component_name[len(REPO_NAME_PREFIX):]
|
||||
|
|
|
|||
|
|
@ -29,12 +29,10 @@ DEFAULTS = {
|
|||
'clone_concurrency': multiprocessing.cpu_count(),
|
||||
'skip_empty': True,
|
||||
'path': os.path.expanduser('~/ccp-repos/'),
|
||||
'hostname': 'git.openstack.org',
|
||||
'port': 443,
|
||||
'protocol': 'https',
|
||||
'project': 'openstack',
|
||||
'username': None,
|
||||
'names': DEFAULT_REPOS,
|
||||
'repos': [{
|
||||
'name': name,
|
||||
'git_url': 'https://git.openstack.org/openstack/{}'.format(name),
|
||||
} for name in DEFAULT_REPOS],
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -47,18 +45,19 @@ SCHEMA = {
|
|||
'clone_concurrency': {'type': 'integer'},
|
||||
'skip_empty': {'type': 'boolean'},
|
||||
'path': {'type': 'string'},
|
||||
'hostname': {'type': 'string'},
|
||||
'port': {'type': 'integer'},
|
||||
'protocol': {'type': 'string'},
|
||||
'project': {'type': 'string'},
|
||||
'username': {'anyOf': [{'type': 'string'}, {'type': 'null'}]},
|
||||
'names': {'type': 'array', 'items': {'type': 'string'}},
|
||||
'repos': {
|
||||
'type': 'array',
|
||||
'items': {
|
||||
'type': 'object',
|
||||
'additionalProperties': False,
|
||||
'required': ['name', 'git_url'],
|
||||
'properties': {
|
||||
'name': {'type': 'string'},
|
||||
'git_url': {'type': 'string'},
|
||||
'git_ref': {'type': 'string'},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for repo in DEFAULT_REPOS:
|
||||
conf_name = repo.replace('-', '_')
|
||||
SCHEMA['repositories']['properties'][conf_name] = \
|
||||
{'anyOf': [{'type': 'string'}, {'type': 'null'}]}
|
||||
DEFAULTS['repositories'][conf_name] = None
|
||||
|
|
|
|||
|
|
@ -14,28 +14,26 @@ LOG = logging.getLogger(__name__)
|
|||
FETCH_TIMEOUT = 2 ** 16 # in seconds
|
||||
|
||||
|
||||
def fetch_repository(repository_name):
|
||||
dest_dir = os.path.join(CONF.repositories.path, repository_name)
|
||||
def fetch_repository(repository_def):
|
||||
name = repository_def['name']
|
||||
dest_dir = os.path.join(CONF.repositories.path, name)
|
||||
if os.path.isdir(dest_dir):
|
||||
LOG.debug('%s was already cloned, skipping', repository_name)
|
||||
LOG.debug('%s was already cloned, skipping', name)
|
||||
return
|
||||
git_url = getattr(CONF.repositories, repository_name.replace('-', '_'))
|
||||
if git_url is None:
|
||||
username = CONF.repositories.username
|
||||
if username is None:
|
||||
username = ''
|
||||
else:
|
||||
username = username + '@'
|
||||
fmt = '{0.protocol}://{1}{0.hostname}:{0.port}/{0.project}/{2}'
|
||||
git_url = fmt.format(CONF.repositories, username, repository_name)
|
||||
LOG.debug('Clonning %s from %s', repository_name, git_url)
|
||||
git.Repo.clone_from(git_url, dest_dir)
|
||||
LOG.info('Cloned %s repo', repository_name)
|
||||
git_url = repository_def['git_url']
|
||||
git_ref = repository_def.get('git_ref')
|
||||
if git_ref:
|
||||
kwargs = {'branch': git_ref}
|
||||
else:
|
||||
kwargs = {}
|
||||
LOG.debug('Clonning %s from %s to %s', name, git_url, dest_dir)
|
||||
git.Repo.clone_from(git_url, dest_dir, **kwargs)
|
||||
LOG.info('Cloned %s repo', name)
|
||||
|
||||
|
||||
def fetch_repositories(repository_names=None):
|
||||
if repository_names is None:
|
||||
repository_names = CONF.repositories.names
|
||||
def fetch_repositories(repository_defs=None):
|
||||
if repository_defs is None:
|
||||
repository_defs = CONF.repositories.repos
|
||||
|
||||
LOG.info('Cloning repositories into %s', CONF.repositories.path)
|
||||
|
||||
|
|
@ -44,9 +42,9 @@ def fetch_repositories(repository_names=None):
|
|||
max_workers=CONF.repositories.clone_concurrency) as executor:
|
||||
future_list = []
|
||||
try:
|
||||
for repository_name in repository_names:
|
||||
for repository_def in repository_defs:
|
||||
future_list.append(executor.submit(
|
||||
fetch_repository, repository_name
|
||||
fetch_repository, repository_def
|
||||
))
|
||||
|
||||
for future in future_list:
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ class TestUtils(base.TestCase):
|
|||
base_dir = os.path.dirname(__file__)
|
||||
|
||||
self.conf.repositories.path = os.path.join(base_dir, "test_repo_dir")
|
||||
self.conf.repositories.names = ["component"]
|
||||
self.conf.repositories.repos = [{"name": "component"}]
|
||||
|
||||
res = (
|
||||
utils.get_deploy_components_info()["keystone"]["service_content"]
|
||||
|
|
@ -73,7 +73,7 @@ class TestUtils(base.TestCase):
|
|||
base_dir = os.path.dirname(__file__)
|
||||
|
||||
self.conf.repositories.path = os.path.join(base_dir, "test_repo_dir")
|
||||
self.conf.repositories.names = ["component"]
|
||||
self.conf.repositories.repos = [{"name": "component"}]
|
||||
|
||||
config.load_component_defaults()
|
||||
|
||||
|
|
@ -111,7 +111,7 @@ class TestUtils(base.TestCase):
|
|||
base_dir = os.path.dirname(__file__)
|
||||
|
||||
self.conf.repositories.path = os.path.join(base_dir, "test_repo_dir")
|
||||
self.conf.repositories.names = ["component"]
|
||||
self.conf.repositories.repos = [{"name": "component"}]
|
||||
|
||||
config.load_component_defaults()
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ class SafeCCPApp(cli.CCPApp):
|
|||
# Debug does magic in cliff, we need it always on
|
||||
parser = super(SafeCCPApp, self).build_option_parser(
|
||||
description, version, argparse_kwargs)
|
||||
parser.set_defaults(debug=True)
|
||||
parser.set_defaults(debug=True, verbosity_level=2)
|
||||
return parser
|
||||
|
||||
def get_fuzzy_matches(self, cmd):
|
||||
|
|
@ -143,7 +143,7 @@ class TestFetch(TestParser):
|
|||
|
||||
def test_parser(self):
|
||||
self._run_app()
|
||||
self.fetch_mock.assert_called_once_with(config.CONF.repositories.names)
|
||||
self.fetch_mock.assert_called_once_with()
|
||||
|
||||
|
||||
class TestCleanup(TestParser):
|
||||
|
|
|
|||
|
|
@ -3,92 +3,45 @@ import os
|
|||
import fixtures
|
||||
from fuel_ccp import fetch
|
||||
from fuel_ccp.tests import base
|
||||
import mock
|
||||
import testscenarios
|
||||
|
||||
|
||||
@mock.patch('git.Repo.clone_from')
|
||||
class TestFetch(testscenarios.WithScenarios, base.TestCase):
|
||||
component_def = {'name': 'compname', 'git_url': 'theurl'}
|
||||
update_def = {}
|
||||
expected_clone_call = None
|
||||
dir_exists = False
|
||||
|
||||
scenarios = [
|
||||
("default", {
|
||||
"option": None,
|
||||
"value": None,
|
||||
"url": "https://git.openstack.org:443/openstack/%s"}),
|
||||
("hostname", {
|
||||
"option": "hostname",
|
||||
"value": "host.name",
|
||||
"url": "https://host.name:443/openstack/%s"}),
|
||||
('username', {
|
||||
"option": "username",
|
||||
"value": "someuser",
|
||||
"url": "https://someuser@git.openstack.org:443/openstack/%s",
|
||||
}),
|
||||
('port', {
|
||||
"option": "port",
|
||||
"value": "9999",
|
||||
'url': "https://git.openstack.org:9999/openstack/%s",
|
||||
}),
|
||||
('protocol', {
|
||||
"option": "protocol",
|
||||
"value": "ssh",
|
||||
'url': "ssh://git.openstack.org:443/openstack/%s",
|
||||
}),
|
||||
('protocol', {
|
||||
"option": "protocol",
|
||||
"value": "http",
|
||||
'url': "http://git.openstack.org:443/openstack/%s",
|
||||
}),
|
||||
('protocol', {
|
||||
"option": "protocol",
|
||||
"value": "git",
|
||||
'url': "git://git.openstack.org:443/openstack/%s",
|
||||
}),
|
||||
('protocol', {
|
||||
"option": "protocol",
|
||||
"value": "https",
|
||||
'url': "https://git.openstack.org:443/openstack/%s",
|
||||
}),
|
||||
('project', {
|
||||
"option": "project",
|
||||
"value": "someproject",
|
||||
'url': "https://git.openstack.org:443/someproject/%s",
|
||||
})
|
||||
('exists', {'dir_exists': True}),
|
||||
]
|
||||
url = None
|
||||
option = None
|
||||
value = None
|
||||
|
||||
def setUp(self):
|
||||
super(TestFetch, self).setUp()
|
||||
# Creating temporaty directory for repos
|
||||
tmp_dir = fixtures.TempDir()
|
||||
tmp_dir.setUp()
|
||||
self.tmp_path = tmp_dir.path
|
||||
self.tmp_path = self.useFixture(fixtures.TempDir()).path
|
||||
self.conf['repositories']['path'] = self.tmp_path
|
||||
# Create temporary directory for openstack-base to not clone it
|
||||
os.mkdir(os.path.join(self.tmp_path, 'ms-openstack-base'))
|
||||
fixture = fixtures.MockPatch('git.Repo.clone_from')
|
||||
self.mock_clone = self.useFixture(fixture).mock
|
||||
|
||||
def test_fetch_default_repositories(self, m_clone):
|
||||
if self.option is not None:
|
||||
self.conf['repositories'][self.option] = self.value
|
||||
self.conf['repositories']['path'] = self.tmp_path
|
||||
components = ['fuel-ccp-debian-base',
|
||||
'fuel-ccp-entrypoint',
|
||||
'fuel-ccp-etcd',
|
||||
'fuel-ccp-glance',
|
||||
'fuel-ccp-horizon',
|
||||
'fuel-ccp-keystone',
|
||||
'fuel-ccp-mariadb',
|
||||
'fuel-ccp-memcached',
|
||||
'fuel-ccp-neutron',
|
||||
'fuel-ccp-nova',
|
||||
'fuel-ccp-rabbitmq',
|
||||
'fuel-ccp-stacklight']
|
||||
expected_calls = [
|
||||
mock.call(
|
||||
self.url % (component),
|
||||
os.path.join(self.tmp_path, component))
|
||||
for component in components]
|
||||
for component, expected_call in zip(components, expected_calls):
|
||||
fetch.fetch_repository(component)
|
||||
self.assertIn(expected_call, m_clone.call_args_list)
|
||||
def test_fetch_repository(self):
|
||||
component_def = self.component_def.copy()
|
||||
component_def.update(self.update_def)
|
||||
|
||||
fixture = fixtures.MockPatch('os.path.isdir')
|
||||
isdir_mock = self.useFixture(fixture).mock
|
||||
isdir_mock.return_value = self.dir_exists
|
||||
|
||||
fetch.fetch_repository(component_def)
|
||||
|
||||
git_path = os.path.join(self.tmp_path, component_def['name'])
|
||||
isdir_mock.assert_called_once_with(git_path)
|
||||
if self.expected_clone_call:
|
||||
git_ref = component_def.get('git_ref')
|
||||
if git_ref:
|
||||
self.mock_clone.assert_called_once_with(
|
||||
'theurl', git_path, branch=git_ref)
|
||||
else:
|
||||
self.mock_clone.assert_called_once_with('theurl', git_path)
|
||||
else:
|
||||
self.mock_clone.assert_not_called()
|
||||
|
|
|
|||
Loading…
Reference in New Issue