From bdb2f2f003ffb83072210dde50b9189bdd9cb09a Mon Sep 17 00:00:00 2001 From: Gauvain Pocentek Date: Sat, 31 May 2014 19:56:26 +0200 Subject: [PATCH] Provide a script to gerenate options changes diff_branches.py generates a listing of the configuration options changes that occured between 2 openstack releases. This involves a few changes in other tools: - the 'dump' subcommand for autohelp.py generates the serialized dict of options - add a special case for the 'bindir' option to avoid getting different default values in different virtual environments - the autohelp-wrapper -e switch builds the needed venv without running autohelp.py commands Change-Id: I80da172b91b8d2f0a15f89f4c812864da2fea471 --- .gitignore | 1 + README.rst | 5 + autogenerate_config_docs/autohelp-wrapper | 25 +- autogenerate_config_docs/autohelp.py | 40 ++- autogenerate_config_docs/diff_branches.py | 283 ++++++++++++++++++++++ 5 files changed, 340 insertions(+), 14 deletions(-) create mode 100755 autogenerate_config_docs/diff_branches.py diff --git a/.gitignore b/.gitignore index b00fc7c6..42003692 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ ChangeLog # Autohelp autogenerate_config_docs/venv autogenerate_config_docs/sources +autogenerate_config_docs/*-conf-changes-*.xml # sitemap sitemap/sitemap_docs.openstack.org.xml diff --git a/README.rst b/README.rst index 997a2fb4..12ba1907 100644 --- a/README.rst +++ b/README.rst @@ -101,6 +101,11 @@ Release notes ---- * ``openstack-doc-test``: Fix handling of ignore-dir parameter. +* ``autohelp-wrapper``: New tool to simplify the setup of an autohelp.py + environment +* ``diff_branches.py``: Generates a listing of the configuration options + changes that occured between 2 openstack releases. +* ``autohelp.py``: add the 'dump' subcommand 0.15 ---- diff --git a/autogenerate_config_docs/autohelp-wrapper b/autogenerate_config_docs/autohelp-wrapper index 73b393a6..bdb9760e 100755 --- a/autogenerate_config_docs/autohelp-wrapper +++ b/autogenerate_config_docs/autohelp-wrapper @@ -35,6 +35,7 @@ usage() { echo "Options:" echo " -b BRANCH: Work on this branch (defaults to master)" echo " -c: Recreate the virtual environment" + echo " -e PATH: Create the virtualenv in PATH" } setup_venv() { @@ -59,7 +60,7 @@ setup_tools() { get_project openstack-manuals (cd $SOURCESDIR/oslo-incubator && python setup.py install) - pip install GitPython>=0.3.2.RC1 + pip install "GitPython>=0.3.2.RC1" # For some reason the ceilometer installation fails without these 2 # packages pre-installed @@ -86,11 +87,11 @@ setup_tools() { # late in the icehouse release cycle to let the doc team handle the changes # properly in the documentation. if echo $BRANCH | grep -q stable/; then - pip install python-keystoneclient==0.7 + pip install "python-keystoneclient==0.7" fi } -while getopts :b:m:c opt; do +while getopts :b:e:c opt; do case $opt in b) BRANCH=$OPTARG @@ -100,6 +101,10 @@ while getopts :b:m:c opt; do rm -rf $VENVDIR shift ;; + e) + VENVDIR=$OPTARG + shift 2 + ;; \?) usage exit 1 @@ -145,7 +150,16 @@ for project in $PROJECTS; do python setup.py install ) - cd $MANUALSREPO/tools/autogenerate-config-flagmappings + if [ "$ACTION" = "setup" ]; then + continue + fi + + if [ -d $MANUALSREPO/tools/autogenerate-config-flagmappings ]; then + cd $MANUALSREPO/tools/autogenerate-config-flagmappings + else + # for havana + $MANUALSREPO/tools/autogenerate-config-docs + fi case $ACTION in update) @@ -155,8 +169,5 @@ for project in $PROJECTS; do docbook) $AUTOHELP docbook $project -i $SOURCESDIR/$project ;; - setup) - # The work is already done - ;; esac done diff --git a/autogenerate_config_docs/autohelp.py b/autogenerate_config_docs/autohelp.py index db39f8da..662c9316 100755 --- a/autogenerate_config_docs/autohelp.py +++ b/autogenerate_config_docs/autohelp.py @@ -23,6 +23,7 @@ from oslo.config import cfg import argparse import importlib import os +import pickle import re import sys @@ -123,6 +124,13 @@ def import_modules(repo_location, package_name, verbose=0): if verbose >= 2: print(e) continue + except cfg.NoSuchGroupError as e: + """ + If a group doesn't exist, we ignore the import. + """ + if verbose >= 2: + print(e) + continue _register_runtime_opts(module, abs_path, verbose) _run_hook(modname) @@ -205,6 +213,11 @@ class OptionsCache(object): if self._verbose >= 2: print ("Duplicate option name %s" % optname) else: + if opt.name == 'bindir': + venv = os.environ.get('VIRTUAL_ENV') + if venv is not None and opt.default.startswith(venv): + opt.default = opt.default.replace(venv, '/usr/local') + self._opts_by_name[optname] = (group, opt) self._opt_names.append(optname) @@ -250,7 +263,8 @@ class OptionsCache(object): return cmp(x, y) -def write_docbook(package_name, options, verbose=0, target='./'): +def write_docbook(package_name, options, verbose=0, + target='../../doc/common/tables/'): """Write DocBook tables. Prints a docbook-formatted table for every group of options. @@ -323,13 +337,13 @@ def write_docbook(package_name, options, verbose=0, target='./'): groups_file.close() -def write_docbook_rootwrap(package_name, repo, verbose=0, target='./'): +def write_docbook_rootwrap(package_name, repo, verbose=0, + target='../../doc/common/tables/'): """Write a DocBook table for rootwrap options. Prints a docbook-formatted table for options in a project's rootwrap.conf configuration file. """ - # The sample rootwrap.conf path is not the same in all projects. It is # either in etc/ or in etc//, so we check both locations. conffile = os.path.join(repo, 'etc', package_name, 'rootwrap.conf') @@ -445,13 +459,21 @@ def update_flagmappings(package_name, options, verbose=0): print(line) +def dump_options(options): + """Dumps the list of options with their attributes. + + This output is consumed by the diff_branches script. + """ + print(pickle.dumps(options._opts_by_name)) + + def main(): parser = argparse.ArgumentParser( description='Manage flag files, to aid in updating documentation.', usage='%(prog)s [options]') parser.add_argument('subcommand', - help='Action (create, update, verify).', - choices=['create', 'update', 'docbook']) + help='Action (create, update, verify, dump).', + choices=['create', 'update', 'docbook', 'dump']) parser.add_argument('package', help='Name of the top-level package.') parser.add_argument('-v', '--verbose', @@ -466,9 +488,11 @@ def main(): type=str,) parser.add_argument('-o', '--output', dest='target', - help='Directory in which xml files are generated.', + help='Directory or file in which data will be saved.\n' + 'Defaults to ../../doc/common/tables/ ' + 'for "docbook".\n' + 'Defaults to stdout for "dump"', required=False, - default='../../doc/common/tables/', type=str,) args = parser.parse_args() @@ -505,6 +529,8 @@ def main(): write_docbook_rootwrap(package_name, args.repo, verbose=args.verbose, target=args.target) + elif args.subcommand == 'dump': + dump_options(options) if __name__ == "__main__": diff --git a/autogenerate_config_docs/diff_branches.py b/autogenerate_config_docs/diff_branches.py new file mode 100755 index 00000000..5be59d3b --- /dev/null +++ b/autogenerate_config_docs/diff_branches.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# A collection of tools for working with flags from OpenStack +# packages and documentation. +# +# For an example of usage, run this program with the -h switch. +# +import argparse +import os +import pickle +import subprocess +import sys + +import git +from lxml import etree + + +PROJECTS = ['ceilometer', 'cinder', 'glance', 'heat', 'keystone', 'neutron', + 'nova', 'trove'] + + +def setup_venv(branch, novenvupdate): + """Uses the autohelp-wrapper script to generate a virtualenv for a given + branch. + """ + dirname = os.path.join('venv', branch.replace('/', '_')) + if novenvupdate and os.path.exists(dirname): + return + if not os.path.exists('venv'): + os.mkdir('venv') + args = ["./autohelp-wrapper", "-b", branch, "-e", dirname, "setup"] + if subprocess.call(args) != 0: + print("Impossible to create the %s environment." % branch) + sys.exit(1) + + +def get_options(project, branch, args): + """Calls the autohelp script in a venv to get the list of known + options. + """ + print("Working on %(project)s (%(branch)s)" % {'project': project, + 'branch': branch}) + # Checkout the required branch + repo_path = os.path.join(args.sources, project) + repo = git.Repo(repo_path) + repo.heads[branch].checkout() + + # And run autohelp script to get a serialized dict of the discovered + # options + dirname = os.path.abspath(os.path.join('venv', branch.replace('/', '_'))) + cmd = ("python autohelp.py dump %(project)s -i %(path)s" % + {'project': project, 'path': repo_path}) + path = os.environ.get("PATH") + bin_path = os.path.abspath(os.path.join(dirname, "bin")) + path = "%s:%s" % (bin_path, path) + serialized = subprocess.check_output(cmd, shell=True, + env={'VIRTUAL_ENV': dirname, + 'PATH': path}) + return pickle.loads(serialized) + + +def _cmpopts(x, y): + """Compare to option names. + + The options can be of 2 forms: option_name or group/option_name. Options + without a group always comes first. Options are sorted alphabetically + inside a group. + """ + if '/' in x and '/' in y: + prex = x[:x.find('/')] + prey = y[:y.find('/')] + if prex != prey: + return cmp(prex, prey) + return cmp(x, y) + elif '/' in x: + return 1 + elif '/' in y: + return -1 + else: + return cmp(x, y) + + +def dbk_append_table(parent, title, cols): + """Create a docbook table and append it to `parent`. + + :param parent: the element to which the table is added + :param title: the table title + :param cols: the number of columns in this table + """ + table = etree.Element("table") + parent.append(table) + caption = etree.Element("caption") + caption.text = title + table.append(caption) + for i in range(cols): + # We cast to int for python 3 + width = "%d%%" % int(100 / cols) + table.append(etree.Element("col", width=width)) + return table + + +def dbk_append_row(parent, cells): + """Append a row to a table. + + :param parent: the table + :param cells: a list of strings, one string per column + """ + tr = etree.Element("tr") + for text in cells: + td = etree.Element("td") + td.text = str(text) + tr.append(td) + parent.append(tr) + + +def dbk_append_header(parent, cells): + """Append a header to a table. + + :param parent: the table + :param cells: a list of strings, one string per column + """ + thead = etree.Element("thead") + dbk_append_row(thead, cells) + parent.append(thead) + + +def diff(old_list, new_list): + """Compare the old and new lists of options to generate lists of modified + options. + """ + new_opts = [] + changed_default = [] + deprecated_opts = [] + for name, (group, option) in new_list.items(): + # Find the new options + if name not in old_list.viewkeys(): + new_opts.append(name) + + # Find the options for which the default value has changed + elif option.default != old_list[name][1].default: + changed_default.append(name) + + # Find options that have been deprecated in the new release. + # If an option name is a key in the old_list dict, it means that it + # wasn't deprecated. + for deprecated in option.deprecated_opts: + # deprecated_opts is a list which always holds at least 1 invalid + # dict. Forget it. + if deprecated.name is None: + continue + + if deprecated.group in [None, 'DEFAULT']: + full_name = deprecated.name + else: + full_name = deprecated.group + '/' + deprecated.name + + if full_name in old_list.viewkeys(): + deprecated_opts.append((full_name, name)) + + return new_opts, changed_default, deprecated_opts + + +def generate_docbook(project, new_branch, old_list, new_list): + """Generate the diff between the 2 options lists for `project`.""" + new_opts, changed_default, deprecated_opts = diff(old_list, new_list) + + XMLNS = '{http://www.w3.org/XML/1998/namespace}' + DOCBOOKMAP = {None: "http://docbook.org/ns/docbook"} + + section = etree.Element("section", nsmap=DOCBOOKMAP, version="5.0") + id = "%(project)s-conf-changes-%(branch)s" % {'project': project, + 'branch': new_branch} + section.set(XMLNS + 'id', id) + section.append(etree.Comment(" Warning: Do not edit this file. It is " + "automatically generated and your changes " + "will be overwritten. The tool to do so " + "lives in the openstack-doc-tools " + "repository. ")) + title = etree.Element("title") + title.text = "New, updated and deprecated options for %s" % project + section.append(title) + + # New options + table = dbk_append_table(section, "New options", 2) + dbk_append_header(table, ["Option = default value", "(Type) Help string"]) + for name in sorted(new_opts, _cmpopts): + opt = new_list[name][1] + type = opt.__class__.__name__.split('.')[-1] + cells = ["%(name)s = %(default)s" % {'name': name, + 'default': opt.default}, + "(%(type)s) %(help)s" % {'type': type, 'help': opt.help}] + dbk_append_row(table, cells) + + table = dbk_append_table(section, "New default values", 3) + dbk_append_header(table, ["Option", "Previous default value", + "New default value"]) + for name in sorted(changed_default, _cmpopts): + old_default = old_list[name][1].default + new_default = new_list[name][1].default + if isinstance(old_default, list): + old_default = ", ".join(old_default) + if isinstance(new_default, list): + new_default = ", ".join(new_default) + cells = [name, old_default, new_default] + dbk_append_row(table, cells) + + table = dbk_append_table(section, "Deprecated options", 2) + dbk_append_header(table, ["Deprecated option", "New Option"]) + for deprecated, new in deprecated_opts: + dbk_append_row(table, [deprecated, new]) + + return etree.tostring(section, pretty_print=True, xml_declaration=True, + encoding="UTF-8") + + +def main(): + parser = argparse.ArgumentParser( + description='Generate a summary of configuration option changes.', + usage='%(prog)s [options]') + parser.add_argument('old_branch', + help='Name of the old branch.') + parser.add_argument('new_branch', + help='Name of the new branch.') + parser.add_argument('-i', '--input', + dest='sources', + help='Path to a folder containing the git ' + 'repositories.', + required=False, + default='./sources', + type=str,) + parser.add_argument('-o', '--output', + dest='target', + help='Directory or file in which data will be saved.\n' + 'Defaults to "."', + required=False, + default='.', + type=str,) + parser.add_argument('-n', '--no-venv-update', + dest='novenvupdate', + help='Don\'t update the virtual envs.', + required=False, + action='store_true', + default=False,) + args = parser.parse_args() + + # Blacklist trove if we diff between havana and icehouse: autohelp.py fails + # with trove on havana + if args.old_branch == "stable/havana": + PROJECTS.remove('trove') + + setup_venv(args.old_branch, args.novenvupdate) + setup_venv(args.new_branch, args.novenvupdate) + + for project in PROJECTS: + old_list = get_options(project, args.old_branch, args) + new_list = get_options(project, args.new_branch, args) + + release = args.new_branch.replace('stable/', '') + xml = generate_docbook(project, release, old_list, new_list) + filename = ("%(project)s-conf-changes-%(release)s.xml" % + {'project': project, 'release': release}) + if not os.path.exists(args.target): + os.makedirs(args.target) + dest = os.path.join(args.target, filename) + with open(dest, 'w') as fd: + fd.write(xml) + + return 0 + + +if __name__ == "__main__": + sys.exit(main())