From bd612be0b6c5f4709bba493603c59d6707f07b09 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Tue, 22 Aug 2017 09:03:24 -0700 Subject: [PATCH] Zuul v3: Add devstack_local_conf ansible module This writes the contents of a variable dictionary as a devstack localrc file. It sorts the output so that references to other variables may be resolved. Change-Id: Ib9716317333aecffca627f5a38a95d86fc4b320b --- .zuul.yaml | 9 +- playbooks/devstack.yaml | 2 +- playbooks/legacy-pre.yaml | 8 +- playbooks/library/__init__.py | 0 .../tasks/main.yaml | 2 + roles/write-devstack-local-conf/README.rst | 60 +++++- .../defaults/main.yaml | 2 + .../library/devstack_local_conf.py | 185 ++++++++++++++++++ .../write-devstack-local-conf/tasks/main.yaml | 14 +- 9 files changed, 268 insertions(+), 14 deletions(-) create mode 100644 playbooks/library/__init__.py create mode 100644 roles/write-devstack-local-conf/defaults/main.yaml create mode 100644 roles/write-devstack-local-conf/library/devstack_local_conf.py diff --git a/.zuul.yaml b/.zuul.yaml index 2e62f62e..6ee922a5 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -8,6 +8,8 @@ name: devstack parent: multinode description: Base devstack job. + required-projects: + - openstack-dev/devstack pre-run: playbooks/pre post-run: @@ -18,10 +20,9 @@ parent: devstack description: A job which sets something in local.conf vars: - devstack_local_conf_contents: | - [[local|localrc]] - LIBVIRT_TYPE=lxc - NOVA_BACKEND=LVM + devstack_localrc: + LIBVIRT_TYPE: lxc + NOVA_BACKEND: LVM - job: name: devstack-legacy diff --git a/playbooks/devstack.yaml b/playbooks/devstack.yaml index a578a149..81ce3a06 100644 --- a/playbooks/devstack.yaml +++ b/playbooks/devstack.yaml @@ -2,4 +2,4 @@ - hosts: all tasks: - command: ls /opt/stack - - command: cat /tmp/dg-local.conf + - command: cat /opt/stack/devstack/local.conf diff --git a/playbooks/legacy-pre.yaml b/playbooks/legacy-pre.yaml index 634f65cf..a6eb3eab 100644 --- a/playbooks/legacy-pre.yaml +++ b/playbooks/legacy-pre.yaml @@ -1,3 +1,7 @@ - hosts: all - roles: - - write-devstack-local-conf + tasks: + - name: Write job-specific localrc variables + copy: + dest: /tmp/dg-local.conf + content: '{{ devstack_local_conf_contents }}' + when: devstack_local_conf_contents is defined diff --git a/playbooks/library/__init__.py b/playbooks/library/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/roles/setup-devstack-source-dirs/tasks/main.yaml b/roles/setup-devstack-source-dirs/tasks/main.yaml index e801fa73..e6bbae23 100644 --- a/roles/setup-devstack-source-dirs/tasks/main.yaml +++ b/roles/setup-devstack-source-dirs/tasks/main.yaml @@ -15,6 +15,8 @@ - name: Set ownership of repos file: path: '{{ devstack_base_dir }}' + state: directory + recurse: true owner: stack group: stack become: yes diff --git a/roles/write-devstack-local-conf/README.rst b/roles/write-devstack-local-conf/README.rst index d6d398c2..e30dfa1e 100644 --- a/roles/write-devstack-local-conf/README.rst +++ b/roles/write-devstack-local-conf/README.rst @@ -2,6 +2,62 @@ Write the local.conf file for use by devstack **Role Variables** -.. zuul:rolevar:: devstack_local_conf_contents +.. zuul:rolevar:: devstack_base_dir + :default: /opt/stack - A string that should be written to devstack's local.conf. + The devstack base directory. + +.. zuul:rolevar:: devstack_local_conf_path + :default: {{ devstack_base_dir }}/devstack/local.conf + + The path of the local.conf file. + +.. zuul:rolevar:: devstack_localrc + :type: dict + + A dictionary of variables that should be written to the localrc + section of local.conf. The values (which are strings) may contain + bash shell variables, and will be ordered so that variables used by + later entries appear first. + +.. zuul:rolevar:: devstack_local_conf + :type: dict + + A complex argument consisting of nested dictionaries which combine + to form the meta-sections of the local_conf file. The top level is + a dictionary of phases, followed by dictionaries of filenames, then + sections, which finally contain key-value pairs for the INI file + entries in those sections. + + The keys in this dictionary are the devstack phases. + + .. zuul:rolevar:: [phase] + :type: dict + + The keys in this dictionary are the filenames for this phase. + + .. zuul:rolevar:: [filename] + :type: dict + + The keys in this dictionary are the INI sections in this file. + + .. zuul:rolevar:: [section] + :type: dict + + This is a dictionary of key-value pairs which comprise + this section of the INI file. + +.. zuul:rolevar:: devstack_services + :type: dict + + A dictionary mapping service names to boolean values. If the + boolean value is ``false``, a ``disable_service`` line will be + emitted for the service name. If it is ``true``, then + ``enable_service`` will be emitted. All other values are ignored. + +.. zuul:rolevar:: devstack_plugins + :type: dict + + A dictionary mapping a plugin name to a git repo location. If the + location is a non-empty string, then an ``enable_plugin`` line will + be emmitted for the plugin name. diff --git a/roles/write-devstack-local-conf/defaults/main.yaml b/roles/write-devstack-local-conf/defaults/main.yaml new file mode 100644 index 00000000..491fa0fd --- /dev/null +++ b/roles/write-devstack-local-conf/defaults/main.yaml @@ -0,0 +1,2 @@ +devstack_base_dir: /opt/stack +devstack_local_conf_path: "{{ devstack_base_dir }}/devstack/local.conf" diff --git a/roles/write-devstack-local-conf/library/devstack_local_conf.py b/roles/write-devstack-local-conf/library/devstack_local_conf.py new file mode 100644 index 00000000..a49142af --- /dev/null +++ b/roles/write-devstack-local-conf/library/devstack_local_conf.py @@ -0,0 +1,185 @@ +# Copyright (C) 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + + +class VarGraph(object): + # This is based on the JobGraph from Zuul. + + def __init__(self, vars): + self.vars = {} + self._varnames = set() + self._dependencies = {} # dependent_var_name -> set(parent_var_names) + for k, v in vars.items(): + self._varnames.add(k) + for k, v in vars.items(): + self._addVar(k, v) + + bash_var_re = re.compile(r'\$\{?(\w+)') + def getDependencies(self, value): + return self.bash_var_re.findall(value) + + def _addVar(self, key, value): + if key in self.vars: + raise Exception("Variable {} already added".format(key)) + self.vars[key] = value + # Append the dependency information + self._dependencies.setdefault(key, set()) + try: + for dependency in self.getDependencies(value): + if dependency == key: + # A variable is allowed to reference itself; no + # dependency link needed in that case. + continue + if dependency not in self._varnames: + # It's not necessary to create a link for an + # external variable. + continue + # Make sure a circular dependency is never created + ancestor_vars = self._getParentVarNamesRecursively( + dependency, soft=True) + ancestor_vars.add(dependency) + if any((key == anc_var) for anc_var in ancestor_vars): + raise Exception("Dependency cycle detected in var {}". + format(key)) + self._dependencies[key].add(dependency) + except Exception: + del self.vars[key] + del self._dependencies[key] + raise + + def getVars(self): + ret = [] + keys = sorted(self.vars.keys()) + seen = set() + for key in keys: + dependencies = self.getDependentVarsRecursively(key) + for var in dependencies + [key]: + if var not in seen: + ret.append((var, self.vars[var])) + seen.add(var) + return ret + + def getDependentVarsRecursively(self, parent_var): + dependent_vars = [] + + current_dependent_vars = self._dependencies[parent_var] + for current_var in current_dependent_vars: + if current_var not in dependent_vars: + dependent_vars.append(current_var) + for dep in self.getDependentVarsRecursively(current_var): + if dep not in dependent_vars: + dependent_vars.append(dep) + return dependent_vars + + def _getParentVarNamesRecursively(self, dependent_var, soft=False): + all_parent_vars = set() + vars_to_iterate = set([dependent_var]) + while len(vars_to_iterate) > 0: + current_var = vars_to_iterate.pop() + current_parent_vars = self._dependencies.get(current_var) + if current_parent_vars is None: + if soft: + current_parent_vars = set() + else: + raise Exception("Dependent var {} not found: ".format( + dependent_var)) + new_parent_vars = current_parent_vars - all_parent_vars + vars_to_iterate |= new_parent_vars + all_parent_vars |= new_parent_vars + return all_parent_vars + + +class LocalConf(object): + + def __init__(self, localrc, localconf, services, plugins): + self.localrc = [] + self.meta_sections = {} + if plugins: + self.handle_plugins(plugins) + if services: + self.handle_services(services) + if localrc: + self.handle_localrc(localrc) + if localconf: + self.handle_localconf(localconf) + + def handle_plugins(self, plugins): + for k, v in plugins.items(): + if v: + self.localrc.append('enable_plugin {} {}'.format(k, v)) + + def handle_services(self, services): + for k, v in services.items(): + if v is False: + self.localrc.append('disable_service {}'.format(k)) + elif v is True: + self.localrc.append('enable_service {}'.format(k)) + + def handle_localrc(self, localrc): + vg = VarGraph(localrc) + for k, v in vg.getVars(): + self.localrc.append('{}={}'.format(k, v)) + + def handle_localconf(self, localconf): + for phase, phase_data in localconf.items(): + for fn, fn_data in phase_data.items(): + ms_name = '[[{}|{}]]'.format(phase, fn) + ms_data = [] + for section, section_data in fn_data.items(): + ms_data.append('[{}]'.format(section)) + for k, v in section_data.items(): + ms_data.append('{} = {}'.format(k, v)) + ms_data.append('') + self.meta_sections[ms_name] = ms_data + + def write(self, path): + with open(path, 'w') as f: + f.write('[[local|localrc]]\n') + f.write('\n'.join(self.localrc)) + f.write('\n\n') + for section, lines in self.meta_sections.items(): + f.write('{}\n'.format(section)) + f.write('\n'.join(lines)) + + +def main(): + module = AnsibleModule( + argument_spec=dict( + plugins=dict(type='dict'), + services=dict(type='dict'), + localrc=dict(type='dict'), + local_conf=dict(type='dict'), + path=dict(type='str'), + ) + ) + + p = module.params + lc = LocalConf(p.get('localrc'), + p.get('local_conf'), + p.get('services'), + p.get('plugins')) + lc.write(p['path']) + + module.exit_json() + + +from ansible.module_utils.basic import * # noqa +from ansible.module_utils.basic import AnsibleModule + +if __name__ == '__main__': + main() diff --git a/roles/write-devstack-local-conf/tasks/main.yaml b/roles/write-devstack-local-conf/tasks/main.yaml index 2595f01c..1d67616d 100644 --- a/roles/write-devstack-local-conf/tasks/main.yaml +++ b/roles/write-devstack-local-conf/tasks/main.yaml @@ -1,5 +1,9 @@ -- name: Write job-specific localrc variables - copy: - dest: /tmp/dg-local.conf - content: '{{ devstack_local_conf_contents }}' - when: devstack_local_conf_contents is defined +- name: Write a job-specific local_conf file + become: true + become_user: stack + devstack_local_conf: + path: "{{ devstack_local_conf_path }}" + plugins: "{{ devstack_plugins|default(omit) }}" + services: "{{ devstack_services|default(omit) }}" + localrc: "{{ devstack_localrc|default(omit) }}" + local_conf: "{{ devstack_local_conf|default(omit) }}"