x
/
fuel-ccp
Code Issues Proposed changes

Fetch git repositories 

Add the fetching mechanism which is looking for git repositories
in place specified in config and (optionally) clones them if
they're not present.

Change-Id: I16d388476ee4b95b8cde28c5a20f79548cd74138
This commit is contained in:
Michal Rostecki 2016-05-09 11:30:47 +02:00
parent d24637b1e0
commit 519427562b
9 changed files with 334 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
microservices/cli.py Normal file
View File

@ -0,0 +1,34 @@
import sys
from oslo_config import cfg
from oslo_log import log as logging
from microservices import fetch
CONF = cfg.CONF
CONF.import_group('repositories', 'microservices.config.repositories')
CONF.import_opt('action', 'microservices.config.cli')
def do_build():
if CONF.repositories.clone:
fetch.fetch_repositories(components=CONF.action.components)
# TODO(mrostecki): implement build
def do_fetch():
fetch.fetch_repositories(components=CONF.action.components)
def main():
logging.register_options(CONF)
logging.setup(CONF, 'microservices')
CONF(sys.argv[1:])
func = globals()['do_%s' % CONF.action.name]
func()
if __name__ == '__main__':
main()

0
microservices/config/__init__.py Normal file
View File

16
microservices/config/auth.py Normal file
View File

@ -0,0 +1,16 @@
import getpass
from oslo_config import cfg
CONF = cfg.CONF
auth_opts = [
cfg.StrOpt('gerrit-username',
default=getpass.getuser())
]
auth_opt_group = cfg.OptGroup(name='auth',
title='Authentication data')
CONF.register_group(auth_opt_group)
CONF.register_cli_opts(auth_opts, auth_opt_group)
CONF.register_opts(auth_opts, auth_opt_group)

20
microservices/config/cli.py Normal file
View File

@ -0,0 +1,20 @@
from oslo_config import cfg
CONF = cfg.CONF
def add_parsers(subparsers):
build_action = subparsers.add_parser('build')
build_action.add_argument('-c', '--components',
nargs='+',
help='MCP component to build')
fetch_action = subparsers.add_parser('fetch')
fetch_action.add_argument('-c', '--components',
nargs='+',
help='MCP component to fetch')
CONF.register_cli_opt(cfg.SubCommandOpt('action',
handler=add_parsers))

149
microservices/config/repositories.py Normal file
View File

@ -0,0 +1,149 @@
import os
from oslo_config import cfg
CONF = cfg.CONF
repositories_opts = [
cfg.BoolOpt('clone',
default=True,
help='Automatic cloning of microservices repositories'),
cfg.StrOpt('path',
default=os.path.expanduser('~/microservices-repos/'),
help='Path where the microservice repositories are cloned'),
cfg.ListOpt('components',
default=['ms-debian-base',
'ms-aodh',
'ms-ceilometer',
'ms-ceph',
'ms-cinder',
'ms-designate',
'ms-elasticsearch',
'ms-glance',
'ms-heat',
'ms-heka',
'ms-horizon',
'ms-ironic',
'ms-keystone',
'ms-kibana',
'ms-magnum',
'ms-manila',
'ms-mariadb',
'ms-memcached',
'ms-mistral',
'ms-mongodb',
'ms-murano',
'ms-neutron',
'ms-nova',
'ms-openstack-base',
'ms-openvswitch',
'ms-rabbitmq',
'ms-sahara',
'ms-swift',
'ms-tempest',
'ms-toolbox',
'ms-trove',
'ms-zaqar'],
help='List of repositories'),
cfg.StrOpt('ms-debian-base',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-debian-base'),
cfg.StrOpt('ms-aodh',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-aodh'),
cfg.StrOpt('ms-ceilometer',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-ceilometer'),
cfg.StrOpt('ms-ceph',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-ceph'),
cfg.StrOpt('ms-cinder',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-cinder'),
cfg.StrOpt('ms-designate',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-designate'),
cfg.StrOpt('ms-elasticsearch',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-elasticsearch'),
cfg.StrOpt('ms-glance',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-glance'),
cfg.StrOpt('ms-heat',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-heat'),
cfg.StrOpt('ms-heka',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-heka'),
cfg.StrOpt('ms-horizon',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-horizon'),
cfg.StrOpt('ms-ironic',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-ironic'),
cfg.StrOpt('ms-keystone',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-keystone'),
cfg.StrOpt('ms-kibana',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-kibana'),
cfg.StrOpt('ms-magnum',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-magnum'),
cfg.StrOpt('ms-manila',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-manila'),
cfg.StrOpt('ms-mariadb',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-mariadb'),
cfg.StrOpt('ms-memcached',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-memcached'),
cfg.StrOpt('ms-mistral',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-mistral'),
cfg.StrOpt('ms-mongodb',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-mongodb'),
cfg.StrOpt('ms-murano',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-murano'),
cfg.StrOpt('ms-neutron',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-neutron'),
cfg.StrOpt('ms-nova',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-nova'),
cfg.StrOpt('ms-openstack-base',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-openstack-base'),
cfg.StrOpt('ms-openvswitch',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-openvswitch'),
cfg.StrOpt('ms-rabbitmq',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-rabbitmq'),
cfg.StrOpt('ms-sahara',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-sahara'),
cfg.StrOpt('ms-swift',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-swift'),
cfg.StrOpt('ms-tempest',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-tempest'),
cfg.StrOpt('ms-toolbox',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-toolbox'),
cfg.StrOpt('ms-trove',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-trove'),
cfg.StrOpt('ms-zaqar',
default='ssh://%s@review.fuel-infra.org:29418/'
'nextgen/ms-zaqar')
]
repositories_opt_group = cfg.OptGroup(name='repositories',
title='Git repositories for components')
CONF.register_group(repositories_opt_group)
CONF.register_cli_opts(repositories_opts, repositories_opt_group)
CONF.register_opts(repositories_opts, repositories_opt_group)

29
microservices/fetch.py Normal file
View File

@ -0,0 +1,29 @@
import os
import git
from oslo_config import cfg
from oslo_log import log as logging
CONF = cfg.CONF
CONF.import_group('auth', 'microservices.config.auth')
CONF.import_group('repositories', 'microservices.config.repositories')
LOG = logging.getLogger(__name__)
def fetch_repositories(components=None):
if components is None:
components = CONF.repositories.components
LOG.info('Cloning repositories into %s', CONF.repositories.path)
for component in components:
dest_dir = os.path.join(CONF.repositories.path, component)
if os.path.isdir(dest_dir):
LOG.info('%s was already cloned, skipping', component)
continue
git_url = getattr(CONF.repositories, component.replace('-', '_')) % \
CONF.auth.gerrit_username
git.Repo.clone_from(git_url, dest_dir)
LOG.info('Cloned %s repo', component)

78
microservices/tests/test_fetch.py Normal file
View File

@ -0,0 +1,78 @@
import getpass
import os
import fixtures
import mock
from oslo_config import cfg
from microservices import fetch
from microservices.tests import base
CONF = cfg.CONF
@mock.patch('git.Repo.clone_from')
class TestFetch(base.TestCase):
def setUp(self):
super(TestFetch, self).setUp()
# Creating temporaty directory for repos
tmp_dir = fixtures.TempDir()
tmp_dir.setUp()
self.tmp_path = tmp_dir.path
CONF.set_override('path', self.tmp_path, group='repositories')
# Create temporary directory for openstack-base to not clone it
os.mkdir(os.path.join(self.tmp_path, 'ms-openstack-base'))
def test_fetch_default_repositories(self, m_clone):
fetch.fetch_repositories()
# All repos except ms-openstack-base
components = [
'ms-debian-base',
'ms-aodh',
'ms-ceilometer',
'ms-ceph',
'ms-cinder',
'ms-designate',
'ms-elasticsearch',
'ms-glance',
'ms-heat',
'ms-heka',
'ms-horizon',
'ms-ironic',
'ms-keystone',
'ms-kibana',
'ms-magnum',
'ms-manila',
'ms-mariadb',
'ms-memcached',
'ms-mistral',
'ms-mongodb',
'ms-murano',
'ms-neutron',
'ms-nova',
'ms-openvswitch',
'ms-rabbitmq',
'ms-sahara',
'ms-swift',
'ms-tempest',
'ms-toolbox',
'ms-trove',
'ms-zaqar'
]
username = getpass.getuser()
expected_calls = [
mock.call('ssh://%s@review.fuel-infra.org:29418/nextgen/%s' % (
username, component), os.path.join(self.tmp_path, component))
for component in components
]
self.assertListEqual(expected_calls, m_clone.call_args_list)
def test_fetch_custom_repositories(self, m_clone):
fetch.fetch_repositories(components=['ms-openstack-base', 'ms-nova'])
username = getpass.getuser()
self.assertListEqual([
mock.call('ssh://%s@review.fuel-infra.org:29418/nextgen/ms-nova' %
username, os.path.join(self.tmp_path, 'ms-nova'))
], m_clone.call_args_list)

4
requirements.txt
View File

@ -3,3 +3,7 @@
# process, which may cause wedges in the gate later.
pbr>=1.6
GitPython>=1.0.1 # BSD License (3 clause)
oslo.config>=3.9.0 # Apache-2.0
oslo.log>=1.14.0 # Apache-2.0

4
setup.cfg
View File

@ -23,6 +23,10 @@ classifier =
packages =
microservices
[entry_points]
console_scripts =
mcp-microservices = microservices.cli:main
[build_sphinx]
source-dir = doc/source
build-dir = doc/build