Make it easy to choose OpenStack versions.

Now it is possible to select components versions for the install
using yaml configurations files and specifying them in the
command line:
* Added new option (--origin) to specify components sources
  configuration file to be used.
* Implemented configuration load (repo, branch, tag) during
  component sources download.
* Remove url parsing logic from downloader and make it work
  with passed configuration.
* Removed components 'get_from' options from configurations.
* Added new sources configuration files for master, havana and
  havana-1.

Implements: blueprint flexible-versions
Change-Id: I35665a309bf8203f10588c01a336e2e62d8e4b5e
This commit is contained in:
Stanislav Kudriashev 2013-10-29 16:00:42 +02:00
parent 00e6e30849
commit 6fa834bad3
33 changed files with 283 additions and 137 deletions

View File

@ -64,12 +64,21 @@ def run(args):
if runner_cls.needs_sudo:
ensure_perms()
# Check persona file exists
persona_fn = args.pop('persona_fn')
if not persona_fn:
raise excp.OptionException("No persona file name specified!")
if not sh.isfile(persona_fn):
raise excp.OptionException("Invalid persona file %r specified!" % (persona_fn))
# Check origin file exists
origins_fn = args.pop('origins_fn')
if not origins_fn:
raise excp.OptionException("No origin file name specified!")
if not sh.isfile(origins_fn):
raise excp.OptionException("Invalid origin file %r specified!" % (origins_fn))
args['origins_fn'] = sh.abspth(origins_fn)
# Determine the root directory...
root_dir = sh.abspth(args.pop("dir"))
@ -111,6 +120,7 @@ def run(args):
colorizer.quote(action), colorizer.quote(utils.iso8601()),
colorizer.quote(dist.name))
LOG.info("Using persona: %s", colorizer.quote(persona_fn))
LOG.info("Using origins: %s", colorizer.quote(origins_fn))
LOG.info("In root directory: %s", colorizer.quote(root_dir))
start_time = time.time()

View File

@ -20,6 +20,7 @@ from anvil import settings
from anvil import shell as sh
from anvil import trace as tr
from anvil import utils
import yaml
from anvil.packaging.helpers import pip_helper
@ -35,34 +36,8 @@ class PkgInstallComponent(base.Component):
self.tracewriter = tr.TraceWriter(trace_fn, break_if_there=False)
self.configurator = conf.Configurator(self)
def _get_download_config(self):
return None
def _get_download_location(self):
key = self._get_download_config()
if not key:
return (None, None)
uri = self.get_option(key, default_value='').strip()
if not uri:
raise ValueError(("Could not find uri in config to download "
"from option %s") % (key))
return (uri, self.get_option('app_dir'))
def download(self):
(from_uri, target_dir) = self._get_download_location()
if not from_uri and not target_dir:
return []
else:
uris = [from_uri]
utils.log_iterable(uris, logger=LOG,
header="Downloading from %s uris" % (len(uris)))
sh.mkdirslist(target_dir, tracewriter=self.tracewriter)
# This is used to delete what is downloaded (done before
# fetching to ensure its cleaned up even on download failures)
self.tracewriter.download_happened(target_dir, from_uri)
fetcher = down.GitDownloader(self.distro, from_uri, target_dir)
fetcher.download()
return uris
return []
def list_patches(self, section):
what_patches = self.get_option('patches', section)
@ -82,7 +57,7 @@ class PkgInstallComponent(base.Component):
def patch(self, section):
canon_what_patches = self.list_patches(section)
if canon_what_patches:
(_from_uri, target_dir) = self._get_download_location()
target_dir = self.get_option('app_dir')
patcher.apply_patches(canon_what_patches, target_dir)
def config_params(self, config_fn):
@ -165,9 +140,30 @@ class PythonInstallComponent(PkgInstallComponent):
self.requires_files.append(sh.joinpths(app_dir,
'test-requirements.txt'))
self._egg_info = None
self._origins_fn = kargs['origins_fn']
def _get_download_config(self):
return 'get_from'
def download(self):
"""Download sources needed to build the component, if any."""
target_dir = self.get_option('app_dir')
with open(self._origins_fn, 'r') as fh:
download_cfg = yaml.safe_load(fh.read()).get(self.name, {})
if not target_dir or not download_cfg:
return []
uri = download_cfg.pop('repo', None)
if not uri:
raise ValueError(("Could not find repo uri for %r component from the %r "
"config file." % (self.name, self._origins_fn)))
uris = [uri]
utils.log_iterable(uris, logger=LOG,
header="Downloading from %s uris" % (len(uris)))
sh.mkdirslist(target_dir, tracewriter=self.tracewriter)
# This is used to delete what is downloaded (done before
# fetching to ensure its cleaned up even on download failures)
self.tracewriter.download_happened(target_dir, uri)
down.GitDownloader(uri, target_dir, **download_cfg).download()
return uris
@property
def egg_info(self):

View File

@ -19,7 +19,6 @@ import contextlib
import functools
import re
import urllib2
import urlparse
import progressbar
@ -33,9 +32,9 @@ LOG = logging.getLogger(__name__)
class Downloader(object):
__metaclass__ = abc.ABCMeta
def __init__(self, uri, store_where):
self.uri = uri
self.store_where = store_where
def __init__(self, uri, dst):
self._uri = uri
self._dst = dst
@abc.abstractmethod
def download(self):
@ -43,47 +42,38 @@ class Downloader(object):
class GitDownloader(Downloader):
def __init__(self, distro, uri, store_where):
Downloader.__init__(self, uri, store_where)
self.distro = distro
def __init__(self, uri, dst, **kwargs):
Downloader.__init__(self, uri, dst)
self._branch = kwargs.get('branch', 'master')
self._tag = str(kwargs.get('tag'))
def download(self):
branch = None
tag = None
uri = self.uri
if uri.find("?") != -1:
# If we use urlparser here it doesn't seem to work right??
# TODO(harlowja), why??
(uri, params) = uri.split("?", 1)
params = urlparse.parse_qs(params)
if 'branch' in params:
branch = params['branch'][0].strip()
if 'tag' in params:
tag = params['tag'][0].strip()
uri = uri.strip()
if not branch:
branch = 'master'
if tag:
branch = self._branch
tag = self._tag
if self._tag:
# Avoid 'detached HEAD state' message by moving to a
# $tag-anvil branch for that tag
new_branch = "%s-%s" % (tag, 'anvil')
new_branch = "%s-%s" % (self._tag, 'anvil')
checkout_what = [tag, '-b', new_branch]
else:
# Set it up to track the remote branch correctly
new_branch = branch
checkout_what = ['-t', '-b', new_branch, 'origin/%s' % branch]
if sh.isdir(self.store_where) and sh.isdir(sh.joinpths(self.store_where, '.git')):
LOG.info("Existing git directory located at %s, leaving it alone.", colorizer.quote(self.store_where))
if sh.isdir(self._dst) and sh.isdir(sh.joinpths(self._dst, '.git')):
LOG.info("Existing git directory located at %s, leaving it alone.",
colorizer.quote(self._dst))
# do git clean -xdfq and git reset --hard to undo possible changes
cmd = ["git", "clean", "-xdfq"]
sh.execute(cmd, cwd=self.store_where)
sh.execute(cmd, cwd=self._dst)
cmd = ["git", "reset", "--hard"]
sh.execute(cmd, cwd=self.store_where)
sh.execute(cmd, cwd=self._dst)
cmd = ["git", "fetch", "origin"]
sh.execute(cmd, cwd=self.store_where)
sh.execute(cmd, cwd=self._dst)
else:
LOG.info("Downloading %s (%s) to %s.", colorizer.quote(uri), branch, colorizer.quote(self.store_where))
cmd = ["git", "clone", uri, self.store_where]
LOG.info("Downloading %s (%s) to %s.", colorizer.quote(self._uri),
branch, colorizer.quote(self._dst))
cmd = ["git", "clone", self._uri, self._dst]
sh.execute(cmd)
if tag:
LOG.info("Adjusting to tag %s.", colorizer.quote(tag))
@ -93,35 +83,36 @@ class GitDownloader(Downloader):
# newer git allows branch resetting: git checkout -B $new_branch
# so, all these are for compatibility with older RHEL git
cmd = ["git", "rev-parse", "HEAD"]
git_head = sh.execute(cmd, cwd=self.store_where)[0].strip()
git_head = sh.execute(cmd, cwd=self._dst)[0].strip()
cmd = ["git", "checkout", git_head]
sh.execute(cmd, cwd=self.store_where)
sh.execute(cmd, cwd=self._dst)
cmd = ["git", "branch", "-D", new_branch]
sh.execute(cmd, cwd=self.store_where, check_exit_code=False)
sh.execute(cmd, cwd=self._dst, check_exit_code=False)
cmd = ["git", "checkout"] + checkout_what
sh.execute(cmd, cwd=self.store_where)
sh.execute(cmd, cwd=self._dst)
# NOTE(aababilov): old openstack.common.setup reports all tag that
# contain HEAD as project's version. It breaks all RPM building
# process, so, we will delete all extra tags
cmd = ["git", "tag", "--contains", "HEAD"]
tag_names = [
i
for i in sh.execute(cmd, cwd=self.store_where)[0].splitlines()
for i in sh.execute(cmd, cwd=self._dst)[0].splitlines()
if i and i != tag]
# Making sure we are not removing tag with the same commit reference
# as for a branch. Otherwise this will make repository broken.
cmd = ["git", "show-ref", "--tags", "--dereference"] + tag_names
for line in sh.execute(cmd, cwd=self.store_where)[0].splitlines():
res = re.search("(.+)\s+refs/tags/(.+)\^\{\}$", line)
if res is None:
continue
ref, tag_name = res.groups()
if ref == git_head and tag_name in tag_names:
tag_names.remove(tag_name)
if tag_names:
cmd = ["git", "show-ref", "--tags", "--dereference"] + tag_names
for line in sh.execute(cmd, cwd=self._dst)[0].splitlines():
res = re.search("(.+)\s+refs/tags/(.+)\^\{\}$", line)
if res is None:
continue
ref, tag_name = res.groups()
if ref == git_head and tag_name in tag_names:
tag_names.remove(tag_name)
if tag_names:
LOG.info("Removing tags: %s", colorizer.quote(" ".join(tag_names)))
cmd = ["git", "tag", "-d"] + tag_names
sh.execute(cmd, cwd=self.store_where)
sh.execute(cmd, cwd=self._dst)
class UrlLibDownloader(Downloader):
@ -140,7 +131,8 @@ class UrlLibDownloader(Downloader):
return progressbar.ProgressBar(widgets=widgets, maxval=size)
def download(self):
LOG.info('Downloading using urllib2: %s to %s.', colorizer.quote(self.uri), colorizer.quote(self.store_where))
LOG.info('Downloading using urllib2: %s to %s.',
colorizer.quote(self._uri), colorizer.quote(self._dst))
p_bar = None
def update_bar(progress_bar, bytes_down):
@ -148,7 +140,7 @@ class UrlLibDownloader(Downloader):
progress_bar.update(bytes_down)
try:
with contextlib.closing(urllib2.urlopen(self.uri, timeout=self.timeout)) as conn:
with contextlib.closing(urllib2.urlopen(self._uri, timeout=self.timeout)) as conn:
c_len = conn.headers.get('content-length')
if c_len is not None:
try:
@ -156,9 +148,9 @@ class UrlLibDownloader(Downloader):
p_bar.start()
except ValueError:
pass
with open(self.store_where, 'wb') as ofh:
return (self.store_where, sh.pipe_in_out(conn, ofh,
chunk_cb=functools.partial(update_bar, p_bar)))
with open(self._dst, 'wb') as ofh:
return (self._dst, sh.pipe_in_out(conn, ofh,
chunk_cb=functools.partial(update_bar, p_bar)))
finally:
if p_bar:
p_bar.finish()

View File

@ -163,6 +163,14 @@ def parse(previous_settings=None):
dest="action",
metavar="ACTION",
help="required action to perform: %s" % (_format_list(actions.names())))
base_group.add_option("-o", "--origins",
action="store",
type="string",
dest="origins_fn",
default=sh.joinpths(settings.ORIGINS_DIR, 'master.yaml'),
metavar="FILE",
help="yaml file describing where to get openstack sources "
"from (default: %default)")
base_group.add_option("-j", "--jobs",
action="store",
type="int",
@ -223,6 +231,7 @@ def parse(previous_settings=None):
values['action'] = (options.action or "")
values['jobs'] = options.jobs
values['persona_fn'] = options.persona_fn
values['origins_fn'] = options.origins_fn
values['verbose'] = options.verbose
values['usr_only'] = options.usr_only
values['prompt_for_passwords'] = options.prompt_for_passwords

View File

@ -21,6 +21,7 @@ import os
CONFIG_DIR = 'conf'
COMPONENT_CONF_DIR = os.path.join(CONFIG_DIR, "components")
DISTRO_DIR = os.path.join(CONFIG_DIR, "distros")
MESSAGING_DIR = os.path.join(CONFIG_DIR, "messages")
ORIGINS_DIR = os.path.join(CONFIG_DIR, "origins")
PERSONA_DIR = os.path.join(CONFIG_DIR, "personas")
TEMPLATE_DIR = os.path.join(CONFIG_DIR, "templates")
MESSAGING_DIR = os.path.join(CONFIG_DIR, "messages")

View File

@ -1,7 +1,4 @@
# Settings for component ceilometer-client
---
# Where we download this from...
get_from: "git://github.com/openstack/python-ceilometerclient.git?branch=master"
...

View File

@ -1,7 +1,4 @@
# Settings for component cinder-client
---
# Where we download this from...
get_from: "git://github.com/openstack/python-cinderclient.git?branch=master"
...

View File

@ -1,13 +1,9 @@
# Settings for component cinder
---
# Where we download this from...
get_from: "git://github.com/openstack/cinder.git?branch=master"
# Host and ports for the different cinder services
api_host: "$(auto:ip)"
api_port: 8776
protocol: http
...

View File

@ -1,5 +1,6 @@
# Settings for component db
---
# Where you db is located at and how to access it.
host: localhost
port: 3306

View File

@ -1,7 +1,4 @@
# Settings for component django-openstack-auth
---
# Where we download this from...
get_from: "git://github.com/openstack/django_openstack_auth.git?branch=master"
...

View File

@ -1,5 +1,6 @@
# Settings for component general
---
ip: "$(auto:ip)"
# How many seconds to wait until a service comes online before using it.

View File

@ -1,9 +1,6 @@
# Settings for component glance-client
---
# Where we download this from...
get_from: "git://github.com/openstack/python-glanceclient.git?branch=master"
# These seem to require swift, not always installed...
exclude_tests:
- "test_ssl_cert_mismatch"

View File

@ -1,7 +1,5 @@
# Settings for component glance
---
# Where we download this from...
get_from: "git://github.com/openstack/glance.git?branch=master"
host: "$(auto:ip)"
api_port: 9292
@ -30,4 +28,5 @@ image_cache_dir: "/usr/share/anvil/glance/images"
# Used by install section in the specfile (conflicts with the client binary...)
remove_file: "/bin/rm -rf %{buildroot}/usr/bin/glance"
...

View File

@ -1,7 +1,4 @@
# Settings for component heat-client
---
# Where we download this from...
get_from: "git://github.com/openstack/python-heatclient.git?branch=master"
...

View File

@ -1,6 +1,4 @@
# Settings for component horizon
---
# Where we download this from...
get_from: "git://github.com/openstack/horizon.git?branch=master"
...

View File

@ -1,9 +1,6 @@
# Settings for component keystone-client
---
# Where we download this from...
get_from: "git://github.com/openstack/python-keystoneclient.git?branch=master"
# This code is out of compliance, so skip it...
use_pep8: False

View File

@ -1,7 +1,5 @@
# Settings for component keystone
---
# Where we download this from...
get_from: "git://github.com/openstack/keystone.git?branch=master"
# Where is the keystone auth host at?
auth_host: "$(auto:ip)"

View File

@ -1,6 +1,4 @@
# Settings for component neutron-client
---
get_from: "git://github.com/openstack/python-neutronclient.git?branch=master"
...

View File

@ -1,9 +1,6 @@
# Settings for component neutron-client
---
# Where we download this from...
get_from: "git://github.com/openstack/neutron.git?branch=master"
# Host and ports for the different neutron services
api_host: "$(auto:ip)"
api_port: 9696

View File

@ -1,7 +1,4 @@
# Settings for component nova-client
---
# Where we download this from...
get_from: "git://github.com/openstack/python-novaclient.git?branch=master"
...

View File

@ -1,9 +1,6 @@
# Settings for component nova
---
# Where we download this from...
get_from: "git://github.com/openstack/nova.git?branch=master"
# Host and ports for the different nova services
api_host: "$(auto:ip)"
api_port: 8774

View File

@ -1,7 +1,4 @@
# Settings for component novnc
---
# Where we download this from...
get_from: "git://github.com/kanaka/noVNC.git?branch=master"
...

View File

@ -1,6 +1,4 @@
# Settings for component openstack-client
---
get_from: "git://github.com/openstack/python-openstackclient.git?branch=master"
...

View File

@ -1,6 +1,4 @@
# Settings for component oslo.config
---
get_from: "git://github.com/openstack/oslo.config.git?branch=master"
...

View File

@ -1,6 +1,4 @@
# Settings for component oslo.config
---
get_from: "git://github.com/openstack/oslo-incubator.git?branch=master"
...

View File

@ -1,5 +1,6 @@
# Settings for component qpid
---
# Where is qpid located?
host: "$(auto:ip)"

View File

@ -1,5 +1,6 @@
# Settings for component rabbit-mq
---
# Where is rabbit located?
host: "$(auto:ip)"

View File

@ -1,6 +1,4 @@
# Settings for component swift-client
---
get_from: "git://github.com/openstack/python-swiftclient.git?branch=master"
...

View File

@ -1,7 +1,4 @@
# Settings for component trove-client
---
# Where we download this from...
get_from: "git://github.com/openstack/python-troveclient.git?branch=master"
...

View File

@ -1,7 +1,4 @@
# Settings for component trove
---
# Where we download this from...
get_from: "git://github.com/openstack/trove.git?branch=master"
...

View File

@ -0,0 +1,63 @@
ceilometer-client:
repo: git://github.com/openstack/python-ceilometerclient.git
tag: 1.0.6
cinder-client:
repo: git://github.com/openstack/python-cinderclient.git
tag: 1.0.6
cinder:
repo: git://github.com/openstack/cinder.git
tag: 2013.2
django-openstack-auth:
repo: git://github.com/openstack/django_openstack_auth.git
tag: 1.1.3
glance-client:
repo: git://github.com/openstack/python-glanceclient.git
tag: 0.11.0
glance:
repo: git://github.com/openstack/glance.git
tag: 2013.2
heat-client:
repo: git://github.com/openstack/python-heatclient.git
tag: 0.2.5
horizon:
repo: git://github.com/openstack/horizon.git
tag: 2013.2
keystone-client:
repo: git://github.com/openstack/python-keystoneclient.git
tag: 0.4.1
keystone:
repo: git://github.com/openstack/keystone.git
tag: 2013.2
nova-client:
repo: git://github.com/openstack/python-novaclient.git
tag: 2.15.0
nova:
repo: git://github.com/openstack/nova.git
tag: 2013.2
novnc:
repo: git://github.com/kanaka/noVNC.git
branch: master
openstack-client:
repo: git://github.com/openstack/python-openstackclient.git
tag: 0.2.2
oslo-config:
repo: git://github.com/openstack/oslo.config.git
tag: 1.2.1
oslo-incubator:
repo: git://github.com/openstack/oslo-incubator.git
tag: 2013.2
neutron-client:
repo: git://github.com/openstack/python-neutronclient.git
tag: 2.3.1
neutron:
repo: git://github.com/openstack/neutron.git
tag: 2013.2
swift-client:
repo: git://github.com/openstack/python-swiftclient.git
tag: 1.8.0
trove-client:
repo: git://github.com/openstack/python-troveclient.git
tag: 1.0.3
trove:
repo: git://github.com/openstack/trove.git
tag: 2013.2

63
conf/origins/havana.yaml Normal file
View File

@ -0,0 +1,63 @@
ceilometer-client:
repo: git://github.com/openstack/python-ceilometerclient.git
tag: 1.0.6
cinder-client:
repo: git://github.com/openstack/python-cinderclient.git
tag: 1.0.6
cinder:
repo: git://github.com/openstack/cinder.git
branch: stable/havana
django-openstack-auth:
repo: git://github.com/openstack/django_openstack_auth.git
tag: 1.1.3
glance-client:
repo: git://github.com/openstack/python-glanceclient.git
tag: 0.11.0
glance:
repo: git://github.com/openstack/glance.git
branch: stable/havana
heat-client:
repo: git://github.com/openstack/python-heatclient.git
tag: 0.2.5
horizon:
repo: git://github.com/openstack/horizon.git
branch: stable/havana
keystone-client:
repo: git://github.com/openstack/python-keystoneclient.git
tag: 0.4.1
keystone:
repo: git://github.com/openstack/keystone.git
branch: stable/havana
nova-client:
repo: git://github.com/openstack/python-novaclient.git
tag: 2.15.0
nova:
repo: git://github.com/openstack/nova.git
branch: stable/havana
novnc:
repo: git://github.com/kanaka/noVNC.git
branch: master
openstack-client:
repo: git://github.com/openstack/python-openstackclient.git
tag: 0.2.2
oslo-config:
repo: git://github.com/openstack/oslo.config.git
branch: stable/havana
oslo-incubator:
repo: git://github.com/openstack/oslo-incubator.git
branch: stable/havana
neutron-client:
repo: git://github.com/openstack/python-neutronclient.git
tag: 2.3.1
neutron:
repo: git://github.com/openstack/neutron.git
branch: stable/havana
swift-client:
repo: git://github.com/openstack/python-swiftclient.git
tag: 1.8.0
trove-client:
repo: git://github.com/openstack/python-troveclient.git
tag: 1.0.3
trove:
repo: git://github.com/openstack/trove.git
branch: stable/havana

63
conf/origins/master.yaml Normal file
View File

@ -0,0 +1,63 @@
ceilometer-client:
repo: git://github.com/openstack/python-ceilometerclient.git
branch: master
cinder-client:
repo: git://github.com/openstack/python-cinderclient.git
branch: master
cinder:
repo: git://github.com/openstack/cinder.git
branch: master
django-openstack-auth:
repo: git://github.com/openstack/django_openstack_auth.git
branch: master
glance-client:
repo: git://github.com/openstack/python-glanceclient.git
branch: master
glance:
repo: git://github.com/openstack/glance.git
branch: master
heat-client:
repo: git://github.com/openstack/python-heatclient.git
branch: master
horizon:
repo: git://github.com/openstack/horizon.git
branch: master
keystone-client:
repo: git://github.com/openstack/python-keystoneclient.git
branch: master
keystone:
repo: git://github.com/openstack/keystone.git
branch: master
nova-client:
repo: git://github.com/openstack/python-novaclient.git
branch: master
nova:
repo: git://github.com/openstack/nova.git
branch: master
novnc:
repo: git://github.com/kanaka/noVNC.git
branch: master
openstack-client:
repo: git://github.com/openstack/python-openstackclient.git
branch: master
oslo-config:
repo: git://github.com/openstack/oslo.config.git
branch: master
oslo-incubator:
repo: git://github.com/openstack/oslo-incubator.git
branch: master
neutron-client:
repo: git://github.com/openstack/python-neutronclient.git
branch: master
neutron:
repo: git://github.com/openstack/neutron.git
branch: master
swift-client:
repo: git://github.com/openstack/python-swiftclient.git
branch: master
trove-client:
repo: git://github.com/openstack/python-troveclient.git
branch: master
trove:
repo: git://github.com/openstack/trove.git
branch: master