Automatically set LIBS_FROM_GIT based on required projects

If a project shows up in zuul's required-projects list, add it
to LIBS_FROM_GIT automatically.  This way, when a user specifies
that a job requires a zuul-project, it gets used in testing, but
otherwise, it doesn't (pypi is used instead).

Also add information about what happens behind the scenes for both
LIBS_FROM_GIT and plugin dependencies.

This moves the check performed in check_libs_from_git to
a helper function which is installed for most kinds of
installations.  This means that if someone sets LIBS_FROM_GIT to
"foobar", devstack won't error anymore, as nothing is going to
try to install foobar, therefore the check won't run on that.
However, as we move to automated generation of the local config,
that error is not likely to happen.  This check was originally
added due to an error in the upper-constraints file (where a
constraint name did not match a package name).  This location of
the check would still catch that type of error.

Change-Id: Ifcf3ad008cf42d3d4762cfb3b6c31c93cfeb40db
(cherry picked from commit e1edde38ed)
This commit is contained in:
James E. Blair 2018-03-02 15:05:14 +00:00 committed by Luigi Toscano
parent b11da3f3cf
commit 27331589e2
6 changed files with 110 additions and 34 deletions

View File

@ -435,22 +435,6 @@ function lib_installed_from_git {
[[ -n $(pip list --format=columns 2>/dev/null | awk "/^$safe_name/ {print \$3}") ]]
}
# check that everything that's in LIBS_FROM_GIT was actually installed
# correctly, this helps double check issues with library fat fingering.
function check_libs_from_git {
local lib=""
local not_installed=""
for lib in $(echo ${LIBS_FROM_GIT} | tr "," " "); do
if ! lib_installed_from_git "$lib"; then
not_installed+=" $lib"
fi
done
# if anything is not installed, say what it is.
if [[ -n "$not_installed" ]]; then
die $LINENO "The following LIBS_FROM_GIT were not installed correct: $not_installed"
fi
}
# setup a library by name. If we are trying to use the library from
# git, we'll do a git based install, otherwise we'll punt and the
# library should be installed by a requirements pull from another
@ -561,6 +545,13 @@ function _setup_package_with_constraints_edit {
setup_package $project_dir "$flags" $extras
# If this project is in LIBS_FROM_GIT, verify it was actually installed
# correctly. This helps catch errors caused by constraints mismatches.
if use_library_from_git "$project_dir"; then
if ! lib_installed_from_git "$project_dir"; then
die $LINENO "The following LIBS_FROM_GIT was not installed correctly: $project_dir"
fi
fi
}
# ``pip install -e`` the package, which processes the dependencies

View File

@ -20,6 +20,14 @@ Write the local.conf file for use by devstack
bash shell variables, and will be ordered so that variables used by
later entries appear first.
As a special case, the variable ``LIBS_FROM_GIT`` will be
constructed automatically from the projects which appear in the
``required-projects`` list defined by the job. To instruct
devstack to install a library from source rather than pypi, simply
add that library to the job's ``required-projects`` list. To
override the automatically-generated value, set ``LIBS_FROM_GIT``
in ``devstack_localrc`` to the desired value.
.. zuul:rolevar:: devstack_local_conf
:type: dict
@ -75,3 +83,7 @@ Write the local.conf file for use by devstack
A dictionary mapping a plugin name to a git repo location. If the
location is a non-empty string, then an ``enable_plugin`` line will
be emmitted for the plugin name.
If a plugin declares a dependency on another plugin (via
``plugin_requires`` in the plugin's settings file), this role will
automatically emit ``enable_plugin`` lines in the correct order.

View File

@ -207,17 +207,17 @@ class PluginGraph(DependencyGraph):
class LocalConf(object):
def __init__(self, localrc, localconf, base_services, services, plugins,
base_dir):
base_dir, projects):
self.localrc = []
self.meta_sections = {}
self.plugin_deps = {}
self.base_dir = base_dir
self.projects = projects
if plugins:
self.handle_plugins(plugins)
if services or base_services:
self.handle_services(base_services, services or {})
if localrc:
self.handle_localrc(localrc)
self.handle_localrc(localrc)
if localconf:
self.handle_localconf(localconf)
@ -241,9 +241,22 @@ class LocalConf(object):
self.localrc.append('enable_service {}'.format(k))
def handle_localrc(self, localrc):
vg = VarGraph(localrc)
for k, v in vg.getVars():
self.localrc.append('{}={}'.format(k, v))
lfg = False
if localrc:
vg = VarGraph(localrc)
for k, v in vg.getVars():
self.localrc.append('{}={}'.format(k, v))
if k == 'LIBS_FROM_GIT':
lfg = True
if not lfg and self.projects:
required_projects = []
for project_name, project_info in self.projects.items():
if project_info.get('required'):
required_projects.append(project_info['short_name'])
if required_projects:
self.localrc.append('LIBS_FROM_GIT={}'.format(
','.join(required_projects)))
def handle_localconf(self, localconf):
for phase, phase_data in localconf.items():
@ -277,6 +290,7 @@ def main():
local_conf=dict(type='dict'),
base_dir=dict(type='path'),
path=dict(type='str'),
projects=dict(type='dict'),
)
)
@ -286,7 +300,8 @@ def main():
p.get('base_services'),
p.get('services'),
p.get('plugins'),
p.get('base_dir'))
p.get('base_dir'),
p.get('projects'))
lc.write(p['path'])
module.exit_json()

View File

@ -56,7 +56,8 @@ class TestDevstackLocalConf(unittest.TestCase):
p.get('base_services'),
p.get('services'),
p.get('plugins'),
p.get('base_dir'))
p.get('base_dir'),
p.get('projects'))
lc.write(p['path'])
plugins = []
@ -66,6 +67,7 @@ class TestDevstackLocalConf(unittest.TestCase):
plugins.append(line.split()[1])
self.assertEqual(['bar', 'baz', 'foo'], plugins)
def test_plugin_deps(self):
"Test that plugins with dependencies work"
os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', 'devstack'))
@ -101,20 +103,80 @@ class TestDevstackLocalConf(unittest.TestCase):
plugins=plugins,
base_dir=self.tmpdir,
path=os.path.join(self.tmpdir, 'test.local.conf'))
def test_libs_from_git(self):
"Test that LIBS_FROM_GIT is auto-generated"
projects = {
'git.openstack.org/openstack/nova': {
'required': True,
'short_name': 'nova',
},
'git.openstack.org/openstack/oslo.messaging': {
'required': True,
'short_name': 'oslo.messaging',
},
'git.openstack.org/openstack/devstack-plugin': {
'required': False,
'short_name': 'devstack-plugin',
},
}
p = dict(base_services=[],
base_dir='./test',
path=os.path.join(self.tmpdir, 'test.local.conf'),
projects=projects)
lc = LocalConf(p.get('localrc'),
p.get('local_conf'),
p.get('base_services'),
p.get('services'),
p.get('plugins'),
p.get('base_dir'))
p.get('base_dir'),
p.get('projects'))
lc.write(p['path'])
plugins = []
lfg = None
with open(p['path']) as f:
for line in f:
if line.startswith('enable_plugin'):
plugins.append(line.split()[1])
self.assertEqual(['foo', 'bar'], plugins)
if line.startswith('LIBS_FROM_GIT'):
lfg = line.strip().split('=')[1]
self.assertEqual('nova,oslo.messaging', lfg)
def test_overridelibs_from_git(self):
"Test that LIBS_FROM_GIT can be overridden"
localrc = {'LIBS_FROM_GIT': 'oslo.db'}
projects = {
'git.openstack.org/openstack/nova': {
'required': True,
'short_name': 'nova',
},
'git.openstack.org/openstack/oslo.messaging': {
'required': True,
'short_name': 'oslo.messaging',
},
'git.openstack.org/openstack/devstack-plugin': {
'required': False,
'short_name': 'devstack-plugin',
},
}
p = dict(localrc=localrc,
base_services=[],
base_dir='./test',
path=os.path.join(self.tmpdir, 'test.local.conf'),
projects=projects)
lc = LocalConf(p.get('localrc'),
p.get('local_conf'),
p.get('base_services'),
p.get('services'),
p.get('plugins'),
p.get('base_dir'),
p.get('projects'))
lc.write(p['path'])
lfg = None
with open(p['path']) as f:
for line in f:
if line.startswith('LIBS_FROM_GIT'):
lfg = line.strip().split('=')[1]
self.assertEqual('oslo.db', lfg)
def test_plugin_circular_deps(self):
"Test that plugins with circular dependencies fail"

View File

@ -9,3 +9,4 @@
localrc: "{{ devstack_localrc|default(omit) }}"
local_conf: "{{ devstack_local_conf|default(omit) }}"
base_dir: "{{ devstack_base_dir|default(omit) }}"
projects: "{{ zuul.projects }}"

View File

@ -1403,11 +1403,6 @@ fi
# Check the status of running services
service_check
# ensure that all the libraries we think we installed from git,
# actually were.
check_libs_from_git
# Configure nova cellsv2
# ----------------------