Zuul: support plugin dependencies

Change-Id: I81302e8988fe6498fea9f08ed66f5d0cc1fce161
This commit is contained in:
James E. Blair 2017-11-21 17:05:43 -08:00
parent b3517ca103
commit 6f27fca4a7
5 changed files with 339 additions and 53 deletions

1
.gitignore vendored
View File

@ -3,6 +3,7 @@
*.log
*.log.[1-9]
*.pem
*.pyc
.localrc.auto
.localrc.password
.prereqs

View File

@ -14,16 +14,69 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import re
class VarGraph(object):
class DependencyGraph(object):
# This is based on the JobGraph from Zuul.
def __init__(self):
self._names = set()
self._dependencies = {} # dependent_name -> set(parent_names)
def add(self, name, dependencies):
# Append the dependency information
self._dependencies.setdefault(name, set())
try:
for dependency in dependencies:
# Make sure a circular dependency is never created
ancestors = self._getParentNamesRecursively(
dependency, soft=True)
ancestors.add(dependency)
if name in ancestors:
raise Exception("Dependency cycle detected in {}".
format(name))
self._dependencies[name].add(dependency)
except Exception:
del self._dependencies[name]
raise
def getDependenciesRecursively(self, parent):
dependencies = []
current_dependencies = self._dependencies[parent]
for current in current_dependencies:
if current not in dependencies:
dependencies.append(current)
for dep in self.getDependenciesRecursively(current):
if dep not in dependencies:
dependencies.append(dep)
return dependencies
def _getParentNamesRecursively(self, dependent, soft=False):
all_parent_items = set()
items_to_iterate = set([dependent])
while len(items_to_iterate) > 0:
current_item = items_to_iterate.pop()
current_parent_items = self._dependencies.get(current_item)
if current_parent_items is None:
if soft:
current_parent_items = set()
else:
raise Exception("Dependent item {} not found: ".format(
dependent))
new_parent_items = current_parent_items - all_parent_items
items_to_iterate |= new_parent_items
all_parent_items |= new_parent_items
return all_parent_items
class VarGraph(DependencyGraph):
def __init__(self, vars):
super(VarGraph, self).__init__()
self.vars = {}
self._varnames = set()
self._dependencies = {} # dependent_var_name -> set(parent_var_names)
for k, v in vars.items():
self._varnames.add(k)
for k, v in vars.items():
@ -38,28 +91,21 @@ class VarGraph(object):
raise Exception("Variable {} already added".format(key))
self.vars[key] = value
# Append the dependency information
self._dependencies.setdefault(key, set())
dependencies = set()
for dependency in self.getDependencies(value):
if dependency == key:
# A variable is allowed to reference itself; no
# dependency link needed in that case.
continue
if dependency not in self._varnames:
# It's not necessary to create a link for an
# external variable.
continue
dependencies.add(dependency)
try:
for dependency in self.getDependencies(value):
if dependency == key:
# A variable is allowed to reference itself; no
# dependency link needed in that case.
continue
if dependency not in self._varnames:
# It's not necessary to create a link for an
# external variable.
continue
# Make sure a circular dependency is never created
ancestor_vars = self._getParentVarNamesRecursively(
dependency, soft=True)
ancestor_vars.add(dependency)
if any((key == anc_var) for anc_var in ancestor_vars):
raise Exception("Dependency cycle detected in var {}".
format(key))
self._dependencies[key].add(dependency)
self.add(key, dependencies)
except Exception:
del self.vars[key]
del self._dependencies[key]
raise
def getVars(self):
@ -67,48 +113,105 @@ class VarGraph(object):
keys = sorted(self.vars.keys())
seen = set()
for key in keys:
dependencies = self.getDependentVarsRecursively(key)
dependencies = self.getDependenciesRecursively(key)
for var in dependencies + [key]:
if var not in seen:
ret.append((var, self.vars[var]))
seen.add(var)
return ret
def getDependentVarsRecursively(self, parent_var):
dependent_vars = []
current_dependent_vars = self._dependencies[parent_var]
for current_var in current_dependent_vars:
if current_var not in dependent_vars:
dependent_vars.append(current_var)
for dep in self.getDependentVarsRecursively(current_var):
if dep not in dependent_vars:
dependent_vars.append(dep)
return dependent_vars
class PluginGraph(DependencyGraph):
def __init__(self, base_dir, plugins):
super(PluginGraph, self).__init__()
# The dependency trees expressed by all the plugins we found
# (which may be more than those the job is using).
self._plugin_dependencies = {}
self.loadPluginNames(base_dir)
def _getParentVarNamesRecursively(self, dependent_var, soft=False):
all_parent_vars = set()
vars_to_iterate = set([dependent_var])
while len(vars_to_iterate) > 0:
current_var = vars_to_iterate.pop()
current_parent_vars = self._dependencies.get(current_var)
if current_parent_vars is None:
if soft:
current_parent_vars = set()
else:
raise Exception("Dependent var {} not found: ".format(
dependent_var))
new_parent_vars = current_parent_vars - all_parent_vars
vars_to_iterate |= new_parent_vars
all_parent_vars |= new_parent_vars
return all_parent_vars
self.plugins = {}
self._pluginnames = set()
for k, v in plugins.items():
self._pluginnames.add(k)
for k, v in plugins.items():
self._addPlugin(k, str(v))
def loadPluginNames(self, base_dir):
if base_dir is None:
return
git_roots = []
for root, dirs, files in os.walk(base_dir):
if '.git' not in dirs:
continue
# Don't go deeper than git roots
dirs[:] = []
git_roots.append(root)
for root in git_roots:
devstack = os.path.join(root, 'devstack')
if not (os.path.exists(devstack) and os.path.isdir(devstack)):
continue
settings = os.path.join(devstack, 'settings')
if not (os.path.exists(settings) and os.path.isfile(settings)):
continue
self.loadDevstackPluginInfo(settings)
define_re = re.compile(r'^define_plugin\s+(\w+).*')
require_re = re.compile(r'^plugin_requires\s+(\w+)\s+(\w+).*')
def loadDevstackPluginInfo(self, fn):
name = None
reqs = set()
with open(fn) as f:
for line in f:
m = self.define_re.match(line)
if m:
name = m.group(1)
m = self.require_re.match(line)
if m:
if name == m.group(1):
reqs.add(m.group(2))
if name and reqs:
self._plugin_dependencies[name] = reqs
def getDependencies(self, value):
return self._plugin_dependencies.get(value, [])
def _addPlugin(self, key, value):
if key in self.plugins:
raise Exception("Plugin {} already added".format(key))
self.plugins[key] = value
# Append the dependency information
dependencies = set()
for dependency in self.getDependencies(key):
if dependency == key:
continue
dependencies.add(dependency)
try:
self.add(key, dependencies)
except Exception:
del self.plugins[key]
raise
def getPlugins(self):
ret = []
keys = sorted(self.plugins.keys())
seen = set()
for key in keys:
dependencies = self.getDependenciesRecursively(key)
for plugin in dependencies + [key]:
if plugin not in seen:
ret.append((plugin, self.plugins[plugin]))
seen.add(plugin)
return ret
class LocalConf(object):
def __init__(self, localrc, localconf, base_services, services, plugins):
def __init__(self, localrc, localconf, base_services, services, plugins,
base_dir):
self.localrc = []
self.meta_sections = {}
self.plugin_deps = {}
self.base_dir = base_dir
if plugins:
self.handle_plugins(plugins)
if services or base_services:
@ -119,7 +222,8 @@ class LocalConf(object):
self.handle_localconf(localconf)
def handle_plugins(self, plugins):
for k, v in plugins.items():
pg = PluginGraph(self.base_dir, plugins)
for k, v in pg.getPlugins():
if v:
self.localrc.append('enable_plugin {} {}'.format(k, v))
@ -171,6 +275,7 @@ def main():
services=dict(type='dict'),
localrc=dict(type='dict'),
local_conf=dict(type='dict'),
base_dir=dict(type='path'),
path=dict(type='str'),
)
)
@ -180,14 +285,18 @@ def main():
p.get('local_conf'),
p.get('base_services'),
p.get('services'),
p.get('plugins'))
p.get('plugins'),
p.get('base_dir'))
lc.write(p['path'])
module.exit_json()
from ansible.module_utils.basic import * # noqa
from ansible.module_utils.basic import AnsibleModule
try:
from ansible.module_utils.basic import * # noqa
from ansible.module_utils.basic import AnsibleModule
except ImportError:
pass
if __name__ == '__main__':
main()

View File

@ -0,0 +1,166 @@
# Copyright (C) 2017 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import shutil
import tempfile
import unittest
from devstack_local_conf import LocalConf
from collections import OrderedDict
class TestDevstackLocalConf(unittest.TestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmpdir)
def test_plugins(self):
"Test that plugins without dependencies work"
localrc = {'test_localrc': '1'}
local_conf = {'install':
{'nova.conf':
{'main':
{'test_conf': '2'}}}}
services = {'cinder': True}
# We use ordereddict here to make sure the plugins are in the
# *wrong* order for testing.
plugins = OrderedDict([
('bar', 'git://git.openstack.org/openstack/bar-plugin'),
('foo', 'git://git.openstack.org/openstack/foo-plugin'),
('baz', 'git://git.openstack.org/openstack/baz-plugin'),
])
p = dict(localrc=localrc,
local_conf=local_conf,
base_services=[],
services=services,
plugins=plugins,
base_dir='./test',
path=os.path.join(self.tmpdir, 'test.local.conf'))
lc = LocalConf(p.get('localrc'),
p.get('local_conf'),
p.get('base_services'),
p.get('services'),
p.get('plugins'),
p.get('base_dir'))
lc.write(p['path'])
plugins = []
with open(p['path']) as f:
for line in f:
if line.startswith('enable_plugin'):
plugins.append(line.split()[1])
self.assertEqual(['bar', 'baz', 'foo'], plugins)
def test_plugin_deps(self):
"Test that plugins with dependencies work"
os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', 'devstack'))
os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', '.git'))
os.makedirs(os.path.join(self.tmpdir, 'bar-plugin', 'devstack'))
os.makedirs(os.path.join(self.tmpdir, 'bar-plugin', '.git'))
with open(os.path.join(
self.tmpdir,
'foo-plugin', 'devstack', 'settings'), 'w') as f:
f.write('define_plugin foo\n')
with open(os.path.join(
self.tmpdir,
'bar-plugin', 'devstack', 'settings'), 'w') as f:
f.write('define_plugin bar\n')
f.write('plugin_requires bar foo\n')
localrc = {'test_localrc': '1'}
local_conf = {'install':
{'nova.conf':
{'main':
{'test_conf': '2'}}}}
services = {'cinder': True}
# We use ordereddict here to make sure the plugins are in the
# *wrong* order for testing.
plugins = OrderedDict([
('bar', 'git://git.openstack.org/openstack/bar-plugin'),
('foo', 'git://git.openstack.org/openstack/foo-plugin'),
])
p = dict(localrc=localrc,
local_conf=local_conf,
base_services=[],
services=services,
plugins=plugins,
base_dir=self.tmpdir,
path=os.path.join(self.tmpdir, 'test.local.conf'))
lc = LocalConf(p.get('localrc'),
p.get('local_conf'),
p.get('base_services'),
p.get('services'),
p.get('plugins'),
p.get('base_dir'))
lc.write(p['path'])
plugins = []
with open(p['path']) as f:
for line in f:
if line.startswith('enable_plugin'):
plugins.append(line.split()[1])
self.assertEqual(['foo', 'bar'], plugins)
def test_plugin_circular_deps(self):
"Test that plugins with circular dependencies fail"
os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', 'devstack'))
os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', '.git'))
os.makedirs(os.path.join(self.tmpdir, 'bar-plugin', 'devstack'))
os.makedirs(os.path.join(self.tmpdir, 'bar-plugin', '.git'))
with open(os.path.join(
self.tmpdir,
'foo-plugin', 'devstack', 'settings'), 'w') as f:
f.write('define_plugin foo\n')
f.write('plugin_requires foo bar\n')
with open(os.path.join(
self.tmpdir,
'bar-plugin', 'devstack', 'settings'), 'w') as f:
f.write('define_plugin bar\n')
f.write('plugin_requires bar foo\n')
localrc = {'test_localrc': '1'}
local_conf = {'install':
{'nova.conf':
{'main':
{'test_conf': '2'}}}}
services = {'cinder': True}
# We use ordereddict here to make sure the plugins are in the
# *wrong* order for testing.
plugins = OrderedDict([
('bar', 'git://git.openstack.org/openstack/bar-plugin'),
('foo', 'git://git.openstack.org/openstack/foo-plugin'),
])
p = dict(localrc=localrc,
local_conf=local_conf,
base_services=[],
services=services,
plugins=plugins,
base_dir=self.tmpdir,
path=os.path.join(self.tmpdir, 'test.local.conf'))
with self.assertRaises(Exception):
lc = LocalConf(p.get('localrc'),
p.get('local_conf'),
p.get('base_services'),
p.get('services'),
p.get('plugins'),
p.get('base_dir'))
lc.write(p['path'])
if __name__ == '__main__':
unittest.main()

View File

@ -8,3 +8,4 @@
services: "{{ devstack_services|default(omit) }}"
localrc: "{{ devstack_localrc|default(omit) }}"
local_conf: "{{ devstack_local_conf|default(omit) }}"
base_dir: "{{ devstack_base_dir|default(omit) }}"

View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
TOP=$(cd $(dirname "$0")/.. && pwd)
# Import common functions
source $TOP/functions
source $TOP/tests/unittest.sh
python ./roles/write-devstack-local-conf/library/test.py