diff --git a/doc/source/generator.rst b/doc/source/generator.rst new file mode 100644 index 00000000..32dc972a --- /dev/null +++ b/doc/source/generator.rst @@ -0,0 +1,11 @@ +--------------------- +oslo-config-generator +--------------------- + +.. automodule:: oslo.config.generator + +.. currentmodule:: oslo.config.generator + +.. autofunction:: main +.. autofunction:: generate +.. autofunction:: register_cli_opts diff --git a/doc/source/index.rst b/doc/source/index.rst index 6363c3dc..b41fb81e 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -18,6 +18,7 @@ Contents parser exceptions styleguide + generator Release Notes ============= diff --git a/oslo/config/generator.py b/oslo/config/generator.py new file mode 100644 index 00000000..9962030a --- /dev/null +++ b/oslo/config/generator.py @@ -0,0 +1,340 @@ +# Copyright 2012 SINA Corporation +# Copyright 2014 Cisco Systems, Inc. +# All Rights Reserved. +# Copyright 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +r""" +A sample configuration file generator. + +oslo-config-generator is a utility for generating sample config files. For +example, to generate a sample config file for oslo.messaging you would run:: + + $> oslo-config-generator --namespace oslo.messaging > oslo.messaging.conf + +This generated sample lists all of the available options, along with their help +string, type, deprecated aliases and defaults. + +The --namespace option specifies an entry point name registered under the +'oslo.config.opts' entry point namespace. For example, in oslo.messaging's +setup.cfg we have:: + + [entry_points] + oslo.config.opts = + oslo.messaging = oslo.messaging.opts:list_opts + +The callable referenced by the entry point should take no arguments and return +a list of (group_name, [opt_1, opt_2]) tuples. For example:: + + opts = [ + cfg.StrOpt('foo'), + cfg.StrOpt('bar'), + ] + + cfg.CONF.register_opts(opts, group='blaa') + + def list_opts(): + return [('blaa', opts)] + +You might choose to return a copy of the options so that the return value can't +be modified for nefarious purposes:: + + def list_opts(): + return [('blaa', copy.deepcopy(opts))] + +A single codebase might have multiple programs, each of which use a subset of +the total set of options registered by the codebase. In that case, you can +register multiple entry points:: + + [entry_points] + oslo.config.opts = + nova.common = nova.config:list_common_opts + nova.api = nova.config:list_api_opts + nova.compute = nova.config:list_compute_opts + +and generate a config file specific to each program:: + + $> oslo-config-generator --namespace oslo.messaging \ + --namespace nova.common \ + --namespace nova.api > nova-api.conf + $> oslo-config-generator --namespace oslo.messaging \ + --namespace nova.common \ + --namespace nova.compute > nova-compute.conf + +To make this more convenient, you can use config files to describe your config +files:: + + $> cat > config-generator/api.conf < cat > config-generator/compute.conf < oslo-config-generator --config-file config-generator/api.conf + $> oslo-config-generator --config-file config-generator/compute.conf + +The default runtime values of configuration options are not always the most +suitable values to include in sample config files - for example, rather than +including the IP address or hostname of the machine where the config file +was generated, you might want to include something like '10.0.0.1'. To +facilitate this, applications can supply their own 'sanitizer' function via +the 'oslo.config.sanitizer' entry point namespace. For example:: + + def sanitize_default(self, opt, default_str): + if opt.dest == 'base_dir' and default_str == os.getcwd(): + return '.' + else: + return default_str + +the callable is registered as a entry point:: + + [entry_points] + oslo.config.opts = + myapp = myapp.opts:list_opts + + oslo.config.sanitizer + myapp = myapp.opts:sanitize_default + +before being passed via the --sanitizer command line option: + + $> oslo-config-generator --namespace myapp \ + --sanitizer myapp > myapp.conf +""" + +import logging +import operator +import sys +import textwrap + +from oslo.config import cfg +import stevedore.named + +LOG = logging.getLogger(__name__) + +_generator_opts = [ + cfg.StrOpt('output-file', + help='Path of the file to write to. Defaults to stdout.'), + cfg.IntOpt('wrap-width', + default=70, + help='The maximum length of help lines.'), + cfg.MultiStrOpt('namespace', + help='Option namespace under "oslo.config.opts" in which ' + 'to query for options.'), + cfg.StrOpt('sanitizer', + help='An entry point name under "oslo.config.sanitizer" for a ' + 'sanitize_default(opt, default_str) callable which will ' + 'sanitize the stringified default value of an option ' + 'before outputting it.'), +] + + +def register_cli_opts(conf): + """Register the formatter's CLI options with a ConfigOpts instance. + + Note, this must be done before the ConfigOpts instance is called to parse + the configuration. + + :param conf: a ConfigOpts instance + :raises: DuplicateOptError, ArgsAlreadyParsedError + """ + conf.register_cli_opts(_generator_opts) + + +class _OptFormatter(object): + + """Format configuration option descriptions to a file.""" + + _TYPE_DESCRIPTIONS = { + cfg.StrOpt: 'string value', + cfg.BoolOpt: 'boolean value', + cfg.IntOpt: 'integer value', + cfg.FloatOpt: 'floating point value', + cfg.ListOpt: 'list value', + cfg.DictOpt: 'dict value', + cfg.MultiStrOpt: 'multi valued', + } + + def __init__(self, output_file=None, sanitize_default=None, wrap_width=70): + """Construct an OptFormatter object. + + :param output_file: a writeable file object + :param sanitize_default: a sanitize_default(opt, default_str) callable + :param wrap_width: The maximum length of help lines, 0 to not wrap + """ + self.output_file = output_file or sys.stdout + self.sanitize_default = sanitize_default or (lambda o, d: d) + self.wrap_width = wrap_width + + def format(self, opt): + """Format a description of an option to the output file. + + :param opt: a cfg.Opt instance + """ + if not opt.help: + LOG.warning('"%s" is missing a help string', opt.dest) + + opt_type = self._TYPE_DESCRIPTIONS.get(type(opt), 'unknown type') + + help_text = u'%s(%s)' % (opt.help + ' ' if opt.help else '', opt_type) + if self.wrap_width is not None and self.wrap_width > 0: + lines = [textwrap.fill(help_text, + self.wrap_width, + initial_indent='# ', + subsequent_indent='# ') + '\n'] + else: + lines = ['# ' + help_text + '\n'] + + for d in opt.deprecated_opts: + lines.append('# Deprecated group/name - [%s]/%s\n' % + (d.group or 'DEFAULT', d.name or opt.dest)) + + if opt.default is None: + default_str = '' + elif isinstance(opt, cfg.StrOpt): + default_str = opt.default + elif isinstance(opt, cfg.BoolOpt): + default_str = str(opt.default).lower() + elif (isinstance(opt, cfg.IntOpt) or + isinstance(opt, cfg.FloatOpt)): + default_str = str(opt.default) + elif isinstance(opt, cfg.ListOpt): + default_str = ','.join(opt.default) + elif isinstance(opt, cfg.DictOpt): + sorted_items = sorted(opt.default.items(), + key=operator.itemgetter(0)) + default_str = ','.join(['%s:%s' % i for i in sorted_items]) + elif isinstance(opt, cfg.MultiStrOpt): + default_str = str(opt.default) + else: + LOG.warning('Unknown option type: %s', repr(opt)) + default_str = str(opt.default) + + defaults = [default_str] + if isinstance(opt, cfg.MultiStrOpt) and opt.default: + defaults = opt.default + + for default_str in defaults: + default_str = self.sanitize_default(opt, default_str) + if default_str.strip() != default_str: + default_str = '"%s"' % default_str + lines.append('#%s = %s\n' % (opt.dest, default_str)) + + self.writelines(lines) + + def write(self, s): + """Write an arbitrary string to the output file. + + :param s: an arbitrary string + """ + self.output_file.write(s) + + def writelines(self, l): + """Write an arbitrary sequence of strings to the output file. + + :param l: a list of arbitrary strings + """ + self.output_file.writelines(l) + + +def _get_sanitizer(name): + """Look up a sanitizer entry point name. + + Look up the supplied name under the 'oslo.config.sanitizer' entry point + namespace and return the callable found there. + + :param name: the entry point name, or None + :returns: the callable found, or None + """ + if name is None: + return None + return stevedore.driver.DriverManager('oslo.config.sanitizer', + name=name).driver + + +def _list_opts(namespaces): + """List the options available via the given namespaces. + + :param namespaces: a list of namespaces registered under 'oslo.config.opts' + :returns: a list of (namespace, [(group, [opt_1, opt_2])]) tuples + """ + mgr = stevedore.named.NamedExtensionManager('oslo.config.opts', + names=namespaces, + invoke_on_load=True) + return [(ep.name, ep.obj) for ep in mgr] + + +def generate(conf): + """Generate a sample config file. + + List all of the options available via the namespaces specified in the given + configuration and write a description of them to the specified output file. + + :param conf: a ConfigOpts instance containing the generator's configuration + """ + conf.register_opts(_generator_opts) + + output_file = (open(conf.output_file, 'w') + if conf.output_file else sys.stdout) + + sanitizer = _get_sanitizer(conf.sanitizer) + + formatter = _OptFormatter(output_file=output_file, + sanitize_default=sanitizer, + wrap_width=conf.wrap_width) + + groups = {'DEFAULT': []} + for namespace, listing in _list_opts(conf.namespace): + for group, opts in listing: + if not opts: + continue + namespaces = groups.setdefault(group or 'DEFAULT', []) + namespaces.append((namespace, + dict((opt.dest, opt) for opt in opts))) + + def _output_opts(f, group, namespaces): + f.write('[%s]\n' % group) + for (namespace, opts_by_dest) in sorted(namespaces, + key=operator.itemgetter(0)): + f.write('\n#\n# From %s\n#\n' % namespace) + for opt in sorted(opts_by_dest.values(), + key=operator.attrgetter('dest')): + f.write('\n') + f.format(opt) + + _output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT')) + for group, namespaces in sorted(groups.items(), + key=operator.itemgetter(0)): + formatter.write('\n\n') + _output_opts(formatter, group, namespaces) + + +def main(args=None): + """The main function of oslo-config-generator.""" + logging.basicConfig(level=logging.WARN) + conf = cfg.ConfigOpts() + register_cli_opts(conf) + conf(args) + generate(conf) + + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt index 7b8af463..cb67f292 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ argparse netaddr>=0.7.6 six>=1.7.0 +stevedore>=0.14 diff --git a/setup.cfg b/setup.cfg index fe7fbfc5..7012627f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,10 @@ namespace_packages = setup-hooks = pbr.hooks.setup_hook +[entry_points] +console_scripts = + oslo-config-generator = oslo.config.generator:main + [build_sphinx] source-dir = doc/source build-dir = doc/build diff --git a/tests/test_generator.py b/tests/test_generator.py new file mode 100644 index 00000000..32754085 --- /dev/null +++ b/tests/test_generator.py @@ -0,0 +1,507 @@ +# Copyright 2014 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +import fixtures +import mock +from oslotest import base +from six import moves +import testscenarios + +from oslo.config import cfg +from oslo.config import fixture as config_fixture +from oslo.config import generator + +load_tests = testscenarios.load_tests_apply_scenarios + + +class GeneratorTestCase(base.BaseTestCase): + + opts = { + 'foo': cfg.StrOpt('foo', help='foo option'), + 'bar': cfg.StrOpt('bar', help='bar option'), + 'foo-bar': cfg.StrOpt('foo-bar', help='foobar'), + 'no_help': cfg.StrOpt('no_help'), + 'long_help': cfg.StrOpt('long_help', + help='Lorem ipsum dolor sit amet, consectetur ' + 'adipisicing elit, sed do eiusmod tempor ' + 'incididunt ut labore et dolore magna ' + 'aliqua. Ut enim ad minim veniam, quis ' + 'nostrud exercitation ullamco laboris ' + 'nisi ut aliquip ex ea commodo ' + 'consequat. Duis aute irure dolor in ' + 'reprehenderit in voluptate velit esse ' + 'cillum dolore eu fugiat nulla ' + 'pariatur. Excepteur sint occaecat ' + 'cupidatat non proident, sunt in culpa ' + 'qui officia deserunt mollit anim id est ' + 'laborum.'), + 'deprecated_opt': cfg.StrOpt('bar', + deprecated_name='foobar', + help='deprecated'), + 'deprecated_group': cfg.StrOpt('bar', + deprecated_group='group1', + deprecated_name='foobar', + help='deprecated'), + 'unknown_type': cfg.Opt('unknown_opt', + default=123, + help='unknown'), + 'str_opt': cfg.StrOpt('str_opt', + default='foo bar', + help='a string'), + 'str_opt_with_space': cfg.StrOpt('str_opt', + default=' foo bar ', + help='a string with spaces'), + 'bool_opt': cfg.BoolOpt('bool_opt', + default=False, + help='a boolean'), + 'int_opt': cfg.IntOpt('int_opt', + default=10, + help='an integer'), + 'float_opt': cfg.FloatOpt('float_opt', + default=0.1, + help='a float'), + 'list_opt': cfg.ListOpt('list_opt', + default=['1', '2', '3'], + help='a list'), + 'dict_opt': cfg.DictOpt('dict_opt', + default={'1': 'yes', '2': 'no'}, + help='a dict'), + 'multi_opt': cfg.MultiStrOpt('multi_opt', + default=['1', '2', '3'], + help='multiple strings'), + } + + content_scenarios = [ + ('empty', + dict(opts=[], expected='''[DEFAULT] +''')), + ('single_namespace', + dict(opts=[('test', [(None, [opts['foo']])])], + expected='''[DEFAULT] + +# +# From test +# + +# foo option (string value) +#foo = +''')), + ('multiple_namespaces', + dict(opts=[('test', [(None, [opts['foo']])]), + ('other', [(None, [opts['bar']])])], + expected='''[DEFAULT] + +# +# From other +# + +# bar option (string value) +#bar = + +# +# From test +# + +# foo option (string value) +#foo = +''')), + ('group', + dict(opts=[('test', [('group1', [opts['foo']])])], + expected='''[DEFAULT] + + +[group1] + +# +# From test +# + +# foo option (string value) +#foo = +''')), + ('empty_group', + dict(opts=[('test', [('group1', [])])], + expected='''[DEFAULT] +''')), + ('multiple_groups', + dict(opts=[('test', [('group1', [opts['foo']]), + ('group2', [opts['bar']])])], + expected='''[DEFAULT] + + +[group1] + +# +# From test +# + +# foo option (string value) +#foo = + + +[group2] + +# +# From test +# + +# bar option (string value) +#bar = +''')), + ('group_in_multiple_namespaces', + dict(opts=[('test', [('group1', [opts['foo']])]), + ('other', [('group1', [opts['bar']])])], + expected='''[DEFAULT] + + +[group1] + +# +# From other +# + +# bar option (string value) +#bar = + +# +# From test +# + +# foo option (string value) +#foo = +''')), + ('hyphenated_name', + dict(opts=[('test', [(None, [opts['foo-bar']])])], + expected='''[DEFAULT] + +# +# From test +# + +# foobar (string value) +#foo_bar = +''')), + ('no_help', + dict(opts=[('test', [(None, [opts['no_help']])])], + log_warning=('"%s" is missing a help string', 'no_help'), + expected='''[DEFAULT] + +# +# From test +# + +# (string value) +#no_help = +''')), + ('long_help', + dict(opts=[('test', [(None, [opts['long_help']])])], + expected='''[DEFAULT] + +# +# From test +# + +# Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do +# eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim +# ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut +# aliquip ex ea commodo consequat. Duis aute irure dolor in +# reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla +# pariatur. Excepteur sint occaecat cupidatat non proident, sunt in +# culpa qui officia deserunt mollit anim id est laborum. (string +# value) +#long_help = +''')), + ('long_help_wrap_at_40', + dict(opts=[('test', [(None, [opts['long_help']])])], + wrap_width=40, + expected='''[DEFAULT] + +# +# From test +# + +# Lorem ipsum dolor sit amet, +# consectetur adipisicing elit, sed do +# eiusmod tempor incididunt ut labore et +# dolore magna aliqua. Ut enim ad minim +# veniam, quis nostrud exercitation +# ullamco laboris nisi ut aliquip ex ea +# commodo consequat. Duis aute irure +# dolor in reprehenderit in voluptate +# velit esse cillum dolore eu fugiat +# nulla pariatur. Excepteur sint +# occaecat cupidatat non proident, sunt +# in culpa qui officia deserunt mollit +# anim id est laborum. (string value) +#long_help = +''')), + ('long_help_no_wrapping', + dict(opts=[('test', [(None, [opts['long_help']])])], + wrap_width=0, + expected='''[DEFAULT] + +# +# From test +# + +''' # noqa +'# Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod ' +'tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, ' +'quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo ' +'consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse ' +'cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat ' +'non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. ' +'(string value)' +''' +#long_help = +''')), + ('deprecated', + dict(opts=[('test', [('foo', [opts['deprecated_opt']])])], + expected='''[DEFAULT] + + +[foo] + +# +# From test +# + +# deprecated (string value) +# Deprecated group/name - [DEFAULT]/foobar +#bar = +''')), + ('deprecated_group', + dict(opts=[('test', [('foo', [opts['deprecated_group']])])], + expected='''[DEFAULT] + + +[foo] + +# +# From test +# + +# deprecated (string value) +# Deprecated group/name - [group1]/foobar +#bar = +''')), + ('unknown_type', + dict(opts=[('test', [(None, [opts['unknown_type']])])], + log_warning=('Unknown option type: %s', + repr(opts['unknown_type'])), + expected='''[DEFAULT] + +# +# From test +# + +# unknown (unknown type) +#unknown_opt = 123 +''')), + ('str_opt', + dict(opts=[('test', [(None, [opts['str_opt']])])], + expected='''[DEFAULT] + +# +# From test +# + +# a string (string value) +#str_opt = foo bar +''')), + ('str_opt_with_space', + dict(opts=[('test', [(None, [opts['str_opt_with_space']])])], + expected='''[DEFAULT] + +# +# From test +# + +# a string with spaces (string value) +#str_opt = " foo bar " +''')), + ('bool_opt', + dict(opts=[('test', [(None, [opts['bool_opt']])])], + expected='''[DEFAULT] + +# +# From test +# + +# a boolean (boolean value) +#bool_opt = false +''')), + ('int_opt', + dict(opts=[('test', [(None, [opts['int_opt']])])], + expected='''[DEFAULT] + +# +# From test +# + +# an integer (integer value) +#int_opt = 10 +''')), + ('float_opt', + dict(opts=[('test', [(None, [opts['float_opt']])])], + expected='''[DEFAULT] + +# +# From test +# + +# a float (floating point value) +#float_opt = 0.1 +''')), + ('list_opt', + dict(opts=[('test', [(None, [opts['list_opt']])])], + expected='''[DEFAULT] + +# +# From test +# + +# a list (list value) +#list_opt = 1,2,3 +''')), + ('dict_opt', + dict(opts=[('test', [(None, [opts['dict_opt']])])], + expected='''[DEFAULT] + +# +# From test +# + +# a dict (dict value) +#dict_opt = 1:yes,2:no +''')), + ('multi_opt', + dict(opts=[('test', [(None, [opts['multi_opt']])])], + expected='''[DEFAULT] + +# +# From test +# + +# multiple strings (multi valued) +#multi_opt = 1 +#multi_opt = 2 +#multi_opt = 3 +''')), + ('sanitizer', + dict(opts=[('test', [(None, [opts['str_opt']])])], + sanitizer=lambda o, s: s.replace(' ', 'ish'), + expected='''[DEFAULT] + +# +# From test +# + +# a string (string value) +#str_opt = fooishbar +''')), + ] + + output_file_scenarios = [ + ('stdout', + dict(stdout=True, output_file=None)), + ('output_file', + dict(output_file='sample.conf', stdout=False)), + ] + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios( + cls.content_scenarios, + cls.output_file_scenarios) + + def setUp(self): + super(GeneratorTestCase, self).setUp() + + self.conf = cfg.ConfigOpts() + self.config_fixture = config_fixture.Config(self.conf) + self.config = self.config_fixture.config + self.useFixture(self.config_fixture) + + self.tempdir = self.useFixture(fixtures.TempDir()) + + def _capture_stream(self, stream_name): + self.useFixture(fixtures.MonkeyPatch("sys.%s" % stream_name, + moves.StringIO())) + return getattr(sys, stream_name) + + def _capture_stdout(self): + return self._capture_stream('stdout') + + @mock.patch('stevedore.named.NamedExtensionManager') + @mock.patch('stevedore.driver.DriverManager') + @mock.patch.object(generator, 'LOG') + def test_generate(self, mock_log, driver_mgr, named_mgr): + generator.register_cli_opts(self.conf) + + namespaces = [i[0] for i in self.opts] + self.config(namespace=namespaces) + + wrap_width = getattr(self, 'wrap_width', None) + if wrap_width is not None: + self.config(wrap_width=wrap_width) + + if self.stdout: + stdout = self._capture_stdout() + else: + output_file = self.tempdir.join(self.output_file) + self.config(output_file=output_file) + + mock_eps = [] + for name, opts in self.opts: + mock_ep = mock.Mock() + mock_ep.configure_mock(name=name, obj=opts) + mock_eps.append(mock_ep) + named_mgr.return_value = mock_eps + + sanitizer = getattr(self, 'sanitizer', None) + if sanitizer is not None: + self.config(sanitizer='test_sanitizer') + driver = mock.Mock(driver=sanitizer) + driver_mgr.return_value = driver + + generator.generate(self.conf) + + if self.stdout: + self.assertEqual(self.expected, stdout.getvalue()) + else: + content = open(output_file).read() + self.assertEqual(self.expected, content) + + named_mgr.assert_called_once_with('oslo.config.opts', + names=namespaces, + invoke_on_load=True) + + if sanitizer is not None: + driver_mgr.assert_called_once_with('oslo.config.sanitizer', + name='test_sanitizer') + pass + else: + self.assertFalse(driver_mgr.called) + + log_warning = getattr(self, 'log_warning', None) + if log_warning is not None: + mock_log.warning.assert_called_once_with(*log_warning) + else: + self.assertFalse(mock_log.warning.called) + + +GeneratorTestCase.generate_scenarios()