Merge "Remove swift tooling"
This commit is contained in:
commit
121ddfebd1
|
@ -29,7 +29,7 @@ On Ubuntu:
|
|||
libxml2-dev libxslt1-dev zlib1g-dev \
|
||||
libmysqlclient-dev libpq-dev libffi-dev \
|
||||
libsqlite3-dev libldap2-dev libsasl2-dev \
|
||||
libjpeg-dev liberasurecode-dev
|
||||
libjpeg-dev
|
||||
|
||||
On RHEL 7 and CentOS 7:
|
||||
|
||||
|
@ -41,11 +41,10 @@ On RHEL 7 and CentOS 7:
|
|||
libxml2-devel libxslt-devel zlib-devel \
|
||||
mariadb-devel postgresql-devel libffi-devel \
|
||||
sqlite-devel openldap-devel cyrus-sasl-devel \
|
||||
libjpeg-turbo-devel liberasurecode-devel gcc git
|
||||
libjpeg-turbo-devel gcc git
|
||||
|
||||
.. note::
|
||||
* libjpeg is needed for ironic
|
||||
* liberasurecode is needed for swift
|
||||
|
||||
The workflow is:
|
||||
|
||||
|
@ -111,19 +110,11 @@ to work on another branch:
|
|||
which will be left untouched (no ``git branch``, no ``git update``).
|
||||
|
||||
|
||||
Updating swift options
|
||||
----------------------
|
||||
|
||||
Swift configuration tables are generated using the ``extract_swift_flags.py``
|
||||
script. This script doesn't use a mapping file, but organize the tables using
|
||||
the various configuration files and sections. Most of the options must be
|
||||
described manually at the moment.
|
||||
|
||||
Generate configuration difference
|
||||
---------------------------------
|
||||
|
||||
To generate "New, updated, and deprecated options" for each service,
|
||||
run diff_branches.py. For example:
|
||||
run ``diff_branches.py``. For example:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
|
|
|
@ -19,11 +19,10 @@ SOURCESDIR=$HERE/sources
|
|||
MANUALSREPO=$SOURCESDIR/openstack-manuals
|
||||
MAPPINGS_DIR=$MANUALSREPO/tools/autogenerate-config-flagmappings
|
||||
AUTOHELP="python $HERE/autohelp.py"
|
||||
EXTRACT_SWIFT="python $HERE/extract_swift_flags.py"
|
||||
GITBASE=${GITBASE:-git://git.openstack.org/openstack}
|
||||
GITPROJ=${GITPROJ:-git://git.openstack.org/openstack}
|
||||
PROJECTS="aodh ceilometer cinder glance heat ironic keystone manila \
|
||||
murano neutron nova sahara senlin swift trove zaqar"
|
||||
murano neutron nova sahara senlin trove zaqar"
|
||||
MANUALS_PROJECTS="openstack-manuals"
|
||||
BRANCH=master
|
||||
FAST=0
|
||||
|
@ -254,30 +253,21 @@ for project in $PROJECTS; do
|
|||
|
||||
case $ACTION in
|
||||
update)
|
||||
[ "$project" = "swift" ] && continue
|
||||
$AUTOHELP update $project -i $SOURCESDIR/$project/$project \
|
||||
$extra_flags $AUTOOPT
|
||||
mv $project.flagmappings.new $project.flagmappings
|
||||
;;
|
||||
rst)
|
||||
if [ "$project" = "swift" ]; then
|
||||
$EXTRACT_SWIFT rst -m $MANUALSREPO -s $SOURCESDIR/swift
|
||||
else
|
||||
$AUTOHELP rst $project -i $SOURCESDIR/$project/$project \
|
||||
$extra_flags $AUTOOPT
|
||||
fi
|
||||
;;
|
||||
dump)
|
||||
if [ $QUIET -eq 1 ]; then
|
||||
exec 1>&3
|
||||
exec 2>&4
|
||||
fi
|
||||
if [ "$project" = "swift" ]; then
|
||||
$EXTRACT_SWIFT dump -m $MANUALSREPO -s $SOURCESDIR/swift
|
||||
else
|
||||
$AUTOHELP dump $project -i $SOURCESDIR/$project/$project \
|
||||
$extra_flags $AUTOOPT
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
|
|
@ -26,8 +26,7 @@ import jinja2
|
|||
|
||||
|
||||
PROJECTS = ['aodh', 'ceilometer', 'cinder', 'glance', 'heat', 'ironic',
|
||||
'keystone', 'manila', 'neutron', 'nova', 'sahara', 'swift',
|
||||
'trove']
|
||||
'keystone', 'manila', 'neutron', 'nova', 'sahara', 'trove']
|
||||
MASTER_RELEASE = 'Ocata'
|
||||
CODENAME_TITLE = {'aodh': 'Alarming',
|
||||
'ceilometer': 'Telemetry',
|
||||
|
@ -42,7 +41,6 @@ CODENAME_TITLE = {'aodh': 'Alarming',
|
|||
'nova': 'Compute',
|
||||
'sahara': 'Data Processing service',
|
||||
'senlin': 'Clustering service',
|
||||
'swift': 'Object Storage service',
|
||||
'trove': 'Database service',
|
||||
'zaqar': 'Message service'}
|
||||
|
||||
|
@ -165,13 +163,6 @@ def format_option_name(name):
|
|||
# name without a section ('log_dir')
|
||||
return "[DEFAULT] %s" % name
|
||||
|
||||
try:
|
||||
# If we're dealing with swift, we'll have a filename to extract
|
||||
# ('proxy-server|filter:tempurl/use')
|
||||
filename, section = section.split('|')
|
||||
return "%s.conf: [%s] %s" % (filename, section, name)
|
||||
except ValueError:
|
||||
# section but no filename ('database/connection')
|
||||
return "[%s] %s" % (section, name)
|
||||
|
||||
|
||||
|
|
|
@ -1,273 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import pickle
|
||||
import sys
|
||||
|
||||
from docutils import core
|
||||
from docutils import io
|
||||
from docutils import nodes
|
||||
import jinja2
|
||||
from oslo_config import cfg
|
||||
|
||||
from autohelp import OptionsCache # noqa
|
||||
|
||||
# Swift configuration example files live in
|
||||
# swift/etc/*.conf-sample
|
||||
# and contain sections enclosed in [], with
|
||||
# options one per line containing =
|
||||
# and generally only having a single entry
|
||||
# after the equals (the default value)
|
||||
|
||||
|
||||
def parse_line(line):
|
||||
"""Parse a line.
|
||||
|
||||
Takes a line from a swift sample configuration file and attempts
|
||||
to separate the lines with actual configuration option and default
|
||||
value from the rest. Returns None if the line doesn't appear to
|
||||
contain a valid configuration option = default value pair, and
|
||||
a pair of the config and its default if it does.
|
||||
"""
|
||||
|
||||
if '=' not in line:
|
||||
return None
|
||||
temp_line = line.strip('#').strip()
|
||||
config, default = temp_line.split('=', 1)
|
||||
config = config.strip()
|
||||
if ' ' in config and config[0:3] != 'set':
|
||||
if len(default.split()) > 1 or config[0].isupper():
|
||||
return None
|
||||
if len(config) < 2 or '.' in config or '<' in config or '>' in config:
|
||||
return None
|
||||
return config, default.strip()
|
||||
|
||||
|
||||
def get_existing_options(optfiles):
|
||||
"""Parse an existing RST table to compile a list of existing options."""
|
||||
options = {}
|
||||
for optfile in optfiles:
|
||||
input_string = open(optfile).read().replace(':ref:', '')
|
||||
output, pub = core.publish_programmatically(
|
||||
source_class=io.StringInput, source=input_string,
|
||||
source_path=optfile, destination_class=io.NullOutput,
|
||||
destination=None, destination_path='/dev/null', reader=None,
|
||||
reader_name='standalone', parser=None,
|
||||
parser_name='restructuredtext', writer=None, writer_name='null',
|
||||
settings=None, settings_spec=None, settings_overrides=None,
|
||||
config_section=None, enable_exit_status=None)
|
||||
doc = pub.writer.document
|
||||
data = dict(doc.traverse(nodes.row, include_self=False)[1:])
|
||||
for a, b in data.items():
|
||||
option = str(a.traverse(nodes.literal)[0].children[0])
|
||||
helptext = str(b.traverse(nodes.paragraph)[0].children[0])
|
||||
|
||||
if option not in options or 'No help text' in options[option]:
|
||||
# options[option.split('=',1)[0]] = helptext
|
||||
options[option] = helptext.strip()
|
||||
|
||||
return options
|
||||
|
||||
|
||||
def extract_descriptions_from_devref(swift_repo, options):
|
||||
"""Extract descriptions from devref RST files.
|
||||
|
||||
Loop through the devref RST files, looking for lines formatted
|
||||
such that they might contain a description of a particular
|
||||
option.
|
||||
"""
|
||||
|
||||
option_descs = {}
|
||||
rsts = glob.glob(swift_repo + '/doc/source/*.rst')
|
||||
for rst in rsts:
|
||||
rst_file = open(rst, 'r')
|
||||
in_option_block = False
|
||||
prev_option = None
|
||||
for line in rst_file:
|
||||
if 'Option ' in line:
|
||||
in_option_block = True
|
||||
if in_option_block:
|
||||
if '========' in line:
|
||||
in_option_block = False
|
||||
continue
|
||||
if line[0] == ' ' and prev_option is not None:
|
||||
option_descs[prev_option] = (option_descs[prev_option]
|
||||
+ ' ' + line.strip())
|
||||
for option in options:
|
||||
line_parts = line.strip().split(None, 2)
|
||||
if (' ' in line and
|
||||
len(line_parts) == 3 and
|
||||
option == line_parts[0] and
|
||||
line_parts[1] != '=' and
|
||||
option != 'use' and
|
||||
(option not in option_descs or
|
||||
len(option_descs[option]) < len(line_parts[2]))):
|
||||
option_descs[option] = line_parts[2]
|
||||
prev_option = option
|
||||
return option_descs
|
||||
|
||||
|
||||
def write_files(options, manuals_repo):
|
||||
"""Create new reStructuredText tables.
|
||||
|
||||
Writes a set of reStructuredText-formatted tables, one per section in swift
|
||||
configuration files.
|
||||
"""
|
||||
all_options = {}
|
||||
names = options.get_option_names()
|
||||
for full_option in sorted(names, OptionsCache._cmpopts):
|
||||
section, optname = full_option.split('/')
|
||||
oslo_opt = options.get_option(full_option)[1]
|
||||
all_options.setdefault(section, [])
|
||||
|
||||
helptext = oslo_opt.help.strip().replace('\n', ' ')
|
||||
helptext = ' '.join(helptext.split())
|
||||
all_options[section].append((oslo_opt.name,
|
||||
oslo_opt.default,
|
||||
helptext))
|
||||
|
||||
for full_section, options in all_options.items():
|
||||
sample_filename, section = full_section.split('|')
|
||||
tmpl_file = os.path.join(os.path.dirname(__file__),
|
||||
'templates/swift.rst.j2')
|
||||
with open(tmpl_file) as fd:
|
||||
template = jinja2.Template(fd.read(), trim_blocks=True)
|
||||
output = template.render(filename=sample_filename,
|
||||
section=section,
|
||||
options=options)
|
||||
|
||||
tgt = (manuals_repo + '/doc/config-reference/source/tables/' +
|
||||
'swift-' + sample_filename + '-' + section + '.rst')
|
||||
|
||||
with open(tgt, 'w') as fd:
|
||||
fd.write(output)
|
||||
|
||||
|
||||
def read_options(swift_repo, manuals_repo, verbose):
|
||||
"""Find swift configuration options.
|
||||
|
||||
Uses existing tables and swift devref as a source of truth in that order to
|
||||
determine helptext for options found in sample config files.
|
||||
"""
|
||||
|
||||
options = get_existing_options(glob.glob(
|
||||
manuals_repo + '/doc/config-reference/source/tables/swift*rst'))
|
||||
option_descs = extract_descriptions_from_devref(swift_repo, options)
|
||||
conf_samples = glob.glob(swift_repo + '/etc/*conf-sample')
|
||||
for sample in conf_samples:
|
||||
current_section = None
|
||||
sample_file = open(sample, 'r')
|
||||
for line in sample_file:
|
||||
if '[' in line and ']\n' in line and '=' not in line:
|
||||
# It's a header line in the conf file, open a new table file
|
||||
# for this section and close any existing one
|
||||
new_line = line.strip('#').strip()
|
||||
if current_section != new_line:
|
||||
current_section = new_line
|
||||
|
||||
base_section = os.path.basename(sample).split('.conf')[0]
|
||||
extra_section = current_section[1:-1].replace(':', '-')
|
||||
full_section = "%s|%s" % (base_section, extra_section)
|
||||
|
||||
continue
|
||||
|
||||
# All the swift files start with a section, except the rsync
|
||||
# sample. The first items are not important for us.
|
||||
if current_section is None:
|
||||
continue
|
||||
|
||||
# It's a config option line in the conf file, find out the
|
||||
# help text and write to the table file.
|
||||
parsed_line = parse_line(line)
|
||||
if parsed_line is not None:
|
||||
if (parsed_line[0] in options.keys()
|
||||
and 'No help text' not in options[parsed_line[0]]):
|
||||
# use the help text from existing tables
|
||||
option_desc = options[parsed_line[0]]
|
||||
elif parsed_line[0] in option_descs:
|
||||
# use the help text from the devref
|
||||
option_desc = option_descs[parsed_line[0]]
|
||||
else:
|
||||
option_desc = 'No help text available for this option.'
|
||||
if verbose > 0:
|
||||
print(parsed_line[0] + " has no help text")
|
||||
|
||||
# \xa0 is a non-breacking space
|
||||
name = parsed_line[0]
|
||||
option_desc = option_desc.replace(u'\xa0', u' ')
|
||||
default = parsed_line[1]
|
||||
|
||||
o = cfg.StrOpt(name=name, default=default, help=option_desc)
|
||||
try:
|
||||
cfg.CONF.register_opt(o, full_section)
|
||||
except cfg.DuplicateOptError:
|
||||
pass
|
||||
|
||||
|
||||
def dump_options(options):
|
||||
"""Dump the list of options with their attributes.
|
||||
|
||||
This output is consumed by the diff_branches script.
|
||||
"""
|
||||
print(pickle.dumps(options._opts_by_name))
|
||||
|
||||
|
||||
def main():
|
||||
"""Parse and write the Swift configuration options."""
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Update the swift options tables.",
|
||||
usage=("%(prog)s rst|dump [-v] [-s swift_repo] "
|
||||
"[-m manuals_repo]"))
|
||||
parser.add_argument('subcommand',
|
||||
help='Action (rst, dump).',
|
||||
choices=['dump', 'rst'])
|
||||
parser.add_argument('-s', '--swift-repo',
|
||||
dest='swift_repo',
|
||||
help="Location of the swift git repository.",
|
||||
required=False,
|
||||
default="./sources/swift")
|
||||
parser.add_argument('-m', '--manuals-repo',
|
||||
dest='manuals_repo',
|
||||
help="Location of the manuals git repository.",
|
||||
required=False,
|
||||
default="./sources/openstack-manuals")
|
||||
parser.add_argument('-v', '--verbose',
|
||||
action='count',
|
||||
default=0,
|
||||
dest='verbose',
|
||||
required=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Avoid cluttering the pickle output, otherwise it's not usable
|
||||
if args.subcommand == 'dump':
|
||||
args.verbose = 0
|
||||
|
||||
read_options(args.swift_repo,
|
||||
args.manuals_repo,
|
||||
args.verbose)
|
||||
options = OptionsCache()
|
||||
|
||||
if args.subcommand == 'rst':
|
||||
write_files(options, args.manuals_repo)
|
||||
else:
|
||||
options.dump()
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
|
@ -1,15 +0,0 @@
|
|||
..
|
||||
Warning: Do not edit this file. It is automatically generated and your
|
||||
changes will be overwritten. The tool to do so lives in the
|
||||
openstack-doc-tools repository.
|
||||
|
||||
.. list-table:: Description of configuration options for ``[{{ section }}]`` in ``{{ filename }}.conf``
|
||||
:header-rows: 1
|
||||
:class: config-ref-table
|
||||
|
||||
* - Configuration option = Default value
|
||||
- Description
|
||||
{% for option in options %}
|
||||
* - ``{{ option[0] }}`` = ``{{ option[1]|default(' ', true) }}``
|
||||
- {{ option[2] }}
|
||||
{% endfor %}
|
|
@ -0,0 +1,9 @@
|
|||
---
|
||||
upgrade:
|
||||
- |
|
||||
The `extract_swift_flags.py` script has been removed, and the `autohelp.py`
|
||||
and `diff_branches.py` scripts no longer build config file documentation
|
||||
for the swift project. The swift dev team had been manually maintaining
|
||||
their config files in-tree and to avoid duplication, doc and swift have
|
||||
agreed to link the config ref out to the dev docs. As such, there is
|
||||
therefore no reason to keep this tooling around.
|
Loading…
Reference in New Issue