From a29c084cb1bde28f31ab120f9d9c60e16e4014c8 Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Tue, 28 Mar 2017 21:29:53 +0000 Subject: [PATCH] Machine Readable Sample Config Adds the ability for the sample config generator to output the config data in the machine readable formats yaml and json. bp machine-readable-sample-config Change-Id: I236918f0c1da27358aace66914aae5c34afef301 Co-Authored-By: Stephen Finucane --- oslo_config/generator.py | 139 +++++++++- oslo_config/tests/test_generator.py | 260 ++++++++++++++++++ ...adable-sample-config-e8f8ba43ababcf99.yaml | 7 + 3 files changed, 396 insertions(+), 10 deletions(-) create mode 100644 releasenotes/notes/machine-readable-sample-config-e8f8ba43ababcf99.yaml diff --git a/oslo_config/generator.py b/oslo_config/generator.py index d38ba7f..a509d75 100644 --- a/oslo_config/generator.py +++ b/oslo_config/generator.py @@ -24,13 +24,16 @@ Tool for generating a sample configuration file. See """ import collections +import copy import logging import operator import sys import textwrap +import json import pkg_resources import six +import yaml from oslo_config import cfg @@ -61,6 +64,15 @@ _generator_opts = [ default=False, help='Only output summaries of help text to config files. Retain ' 'longer help text for Sphinx documents.'), + cfg.StrOpt( + 'format', + help='Desired format for the output. "ini" is the only one which can ' + 'be used directly with oslo.config. "json" and "yaml" are ' + 'intended for third-party tools that want to write config files ' + 'based on the sample config data.', + default='ini', + choices=['ini', 'json', 'yaml'], + dest='format_'), ] @@ -491,6 +503,108 @@ def _get_groups(conf_ns): return groups +def _build_entry(opt, group, namespace, conf): + """Return a dict representing the passed in opt + + The dict will contain all public attributes of opt, as well as additional + entries for namespace, choices, min, and max. Any DeprecatedOpts + contained in the deprecated_opts member will be converted to a dict with + the format: {'group': , 'name': } + + :param opt: The Opt object to represent as a dict. + :param group: The name of the group containing opt. + :param namespace: The name of the namespace containing opt. + :param conf: The ConfigOpts object containing the options for the + generator tool + """ + entry = {key: value for key, value in opt.__dict__.items() + if not key.startswith('_')} + entry['namespace'] = namespace + # In some types, choices is explicitly set to None. Force it to [] so it + # is always an iterable type. + entry['choices'] = getattr(entry['type'], 'choices', []) or [] + entry['min'] = getattr(entry['type'], 'min', None) + entry['max'] = getattr(entry['type'], 'max', None) + entry['type'] = _format_type_name(entry['type']) + deprecated_opts = [] + for deprecated_opt in entry['deprecated_opts']: + # NOTE(bnemec): opt names with a - are not valid in a config file, + # but it is possible to add a DeprecatedOpt with a - name. We + # want to ignore those as they won't work anyway. + if not deprecated_opt.name or '-' not in deprecated_opt.name: + deprecated_opts.append( + {'group': deprecated_opt.group or group, + 'name': deprecated_opt.name or entry['name'], + }) + entry['deprecated_opts'] = deprecated_opts + return entry + + +def _generate_machine_readable_data(groups, conf): + """Create data structure for machine readable sample config + + Returns a dictionary with the top-level keys 'options', + 'deprecated_options', and 'generator_options'. + + 'options' contains a dict mapping group names to a list of options in + that group. Each option is represented by the result of a call to + _build_entry. Only non-deprecated options are included in this list. + + 'deprecated_options' contains a dict mapping groups names to a list of + opts from that group which were deprecated. + + 'generator_options' is a dict mapping the options for the sample config + generator itself to their values. + + :param groups: A dict of groups as returned by _get_groups. + :param conf: The ConfigOpts object containing the options for the + generator tool + """ + output_data = {'options': {}, + 'deprecated_options': {}, + 'generator_options': {}} + # See _get_groups for details on the structure of group_data + for group_name, group_data in groups.items(): + output_data['options'][group_name] = {'opts': [], 'help': ''} + for namespace in group_data['namespaces']: + for opt in namespace[1]: + if group_data['object']: + output_group = output_data['options'][group_name] + output_group['help'] = group_data['object'].help + entry = _build_entry(opt, group_name, namespace[0], conf) + output_data['options'][group_name]['opts'].append(entry) + # Need copies of the opts because we modify them + for deprecated_opt in copy.deepcopy(entry['deprecated_opts']): + group = deprecated_opt.pop('group') + deprecated_options = output_data['deprecated_options'] + deprecated_options.setdefault(group, []) + deprecated_opt['replacement_name'] = entry['name'] + deprecated_opt['replacement_group'] = group_name + deprecated_options[group].append(deprecated_opt) + output_data['generator_options'] = conf + return output_data + + +def _output_machine_readable(groups, output_file, conf): + """Write a machine readable sample config file + + Take the data returned by _generate_machine_readable_data and write it in + the format specified by the format_ attribute of conf. + + :param groups: A dict of groups as returned by _get_groups. + :param output_file: A file-like object to which the data should be written. + :param conf: The ConfigOpts object containing the options for the + generator tool + """ + output_data = _generate_machine_readable_data(groups, conf) + if conf.format_ == 'yaml': + output_file.write(yaml.safe_dump(output_data, + default_flow_style=False)) + else: + output_file.write(json.dumps(output_data, sort_keys=True)) + output_file.write('\n') + + def generate(conf): """Generate a sample config file. @@ -504,21 +618,26 @@ def generate(conf): output_file = (open(conf.output_file, 'w') if conf.output_file else sys.stdout) - formatter = _OptFormatter(output_file=output_file, - wrap_width=conf.wrap_width) - groups = _get_groups(_list_opts(conf.namespace)) - # Output the "DEFAULT" section as the very first section - _output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT'), conf.minimal, - conf.summarize) + if conf.format_ == 'ini': + formatter = _OptFormatter(output_file=output_file, + wrap_width=conf.wrap_width) - # output all other config sections with groups in alphabetical order - for group, group_data in sorted(groups.items()): - formatter.write('\n\n') - _output_opts(formatter, group, group_data, conf.minimal, + # Output the "DEFAULT" section as the very first section + _output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT'), conf.minimal, conf.summarize) + # output all other config sections with groups in alphabetical order + for group, group_data in sorted(groups.items()): + formatter.write('\n\n') + _output_opts(formatter, group, group_data, conf.minimal, + conf.summarize) + else: + _output_machine_readable(groups, + output_file=output_file, + conf=conf) + def main(args=None): """The main function of oslo-config-generator.""" diff --git a/oslo_config/tests/test_generator.py b/oslo_config/tests/test_generator.py index 3406b63..8b075e9 100644 --- a/oslo_config/tests/test_generator.py +++ b/oslo_config/tests/test_generator.py @@ -954,6 +954,265 @@ class GeneratorTestCase(base.BaseTestCase): self.assertFalse(mock_log.warning.called) +GENERATOR_OPTS = {'format_': 'yaml', + 'minimal': False, + 'namespace': ['test'], + 'output_file': None, + 'summarize': False, + 'wrap_width': 70} + + +class MachineReadableGeneratorTestCase(base.BaseTestCase): + all_opts = GeneratorTestCase.opts + all_groups = GeneratorTestCase.groups + content_scenarios = [ + ('single_namespace', + dict(opts=[('test', [(None, [all_opts['foo']])])], + expected={'deprecated_options': {}, + 'generator_options': GENERATOR_OPTS, + 'options': { + 'DEFAULT': { + 'help': '', + 'opts': [{'advanced': False, + 'choices': [], + 'default': None, + 'deprecated_for_removal': False, + 'deprecated_opts': [], + 'deprecated_reason': None, + 'deprecated_since': None, + 'dest': 'foo', + 'help': 'foo option', + 'max': None, + 'metavar': None, + 'min': None, + 'mutable': False, + 'name': 'foo', + 'namespace': 'test', + 'positional': False, + 'required': False, + 'sample_default': None, + 'secret': False, + 'short': None, + 'type': 'string value'}]}}})), + ('long_help', + dict(opts=[('test', [(None, [all_opts['long_help']])])], + expected={'deprecated_options': {}, + 'generator_options': GENERATOR_OPTS, + 'options': { + 'DEFAULT': { + 'help': '', + 'opts': [{'advanced': False, + 'choices': [], + 'default': None, + 'deprecated_for_removal': False, + 'deprecated_opts': [], + 'deprecated_reason': None, + 'deprecated_since': None, + 'dest': 'long_help', + 'help': all_opts['long_help'].help, + 'max': None, + 'metavar': None, + 'min': None, + 'mutable': False, + 'name': 'long_help', + 'namespace': 'test', + 'positional': False, + 'required': False, + 'sample_default': None, + 'secret': False, + 'short': None, + 'type': 'string value'}]}}})), + ('long_help_pre', + dict(opts=[('test', [(None, [all_opts['long_help_pre']])])], + expected={'deprecated_options': {}, + 'generator_options': GENERATOR_OPTS, + 'options': { + 'DEFAULT': { + 'help': '', + 'opts': [{'advanced': False, + 'choices': [], + 'default': None, + 'deprecated_for_removal': False, + 'deprecated_opts': [], + 'deprecated_reason': None, + 'deprecated_since': None, + 'dest': 'long_help_pre', + 'help': + all_opts['long_help_pre'].help, + 'max': None, + 'metavar': None, + 'min': None, + 'mutable': False, + 'name': 'long_help_pre', + 'namespace': 'test', + 'positional': False, + 'required': False, + 'sample_default': None, + 'secret': False, + 'short': None, + 'type': 'string value'}]}}})), + ('opt_with_DeprecatedOpt', + dict(opts=[('test', [(None, [all_opts['opt_with_DeprecatedOpt']])])], + expected={ + 'deprecated_options': { + 'deprecated': [{'name': 'foo_bar', + 'replacement_group': 'DEFAULT', + 'replacement_name': 'foo-bar'}]}, + 'generator_options': GENERATOR_OPTS, + 'options': { + 'DEFAULT': { + 'help': '', + 'opts': [{ + 'advanced': False, + 'choices': [], + 'default': None, + 'deprecated_for_removal': False, + 'deprecated_opts': [{'group': 'deprecated', + 'name': 'foo_bar'}], + 'deprecated_reason': None, + 'deprecated_since': None, + 'dest': 'foo_bar', + 'help': + all_opts['opt_with_DeprecatedOpt'].help, + 'max': None, + 'metavar': None, + 'min': None, + 'mutable': False, + 'name': 'foo-bar', + 'namespace': 'test', + 'positional': False, + 'required': False, + 'sample_default': None, + 'secret': False, + 'short': None, + 'type': 'boolean value'}]}}})), + ('choices_opt', + dict(opts=[('test', [(None, [all_opts['choices_opt']])])], + expected={'deprecated_options': {}, + 'generator_options': GENERATOR_OPTS, + 'options': { + 'DEFAULT': { + 'help': '', + 'opts': [{'advanced': False, + 'choices': (None, '', 'a', 'b', 'c'), + 'default': 'a', + 'deprecated_for_removal': False, + 'deprecated_opts': [], + 'deprecated_reason': None, + 'deprecated_since': None, + 'dest': 'choices_opt', + 'help': all_opts['choices_opt'].help, + 'max': None, + 'metavar': None, + 'min': None, + 'mutable': False, + 'name': 'choices_opt', + 'namespace': 'test', + 'positional': False, + 'required': False, + 'sample_default': None, + 'secret': False, + 'short': None, + 'type': 'string value'}]}}})), + ('int_opt', + dict(opts=[('test', [(None, [all_opts['int_opt']])])], + expected={'deprecated_options': {}, + 'generator_options': GENERATOR_OPTS, + 'options': { + 'DEFAULT': { + 'help': '', + 'opts': [{'advanced': False, + 'choices': [], + 'default': 10, + 'deprecated_for_removal': False, + 'deprecated_opts': [], + 'deprecated_reason': None, + 'deprecated_since': None, + 'dest': 'int_opt', + 'help': all_opts['int_opt'].help, + 'max': 20, + 'metavar': None, + 'min': 1, + 'mutable': False, + 'name': 'int_opt', + 'namespace': 'test', + 'positional': False, + 'required': False, + 'sample_default': None, + 'secret': False, + 'short': None, + 'type': 'integer value'}]}}})), + ('group_help', + dict(opts=[('test', [(all_groups['group1'], [all_opts['foo']])])], + expected={'deprecated_options': {}, + 'generator_options': GENERATOR_OPTS, + 'options': { + 'DEFAULT': { + 'help': '', + 'opts': [] + }, + 'group1': { + 'help': all_groups['group1'].help, + 'opts': [{'advanced': False, + 'choices': [], + 'default': None, + 'deprecated_for_removal': False, + 'deprecated_opts': [], + 'deprecated_reason': None, + 'deprecated_since': None, + 'dest': 'foo', + 'help': all_opts['foo'].help, + 'max': None, + 'metavar': None, + 'min': None, + 'mutable': False, + 'name': 'foo', + 'namespace': 'test', + 'positional': False, + 'required': False, + 'sample_default': None, + 'secret': False, + 'short': None, + 'type': 'string value'}]}}})), + ] + + def setUp(self): + super(MachineReadableGeneratorTestCase, self).setUp() + + self.conf = cfg.ConfigOpts() + self.config_fixture = config_fixture.Config(self.conf) + self.config = self.config_fixture.config + self.useFixture(self.config_fixture) + + @classmethod + def generate_scenarios(cls): + cls.scenarios = testscenarios.multiply_scenarios( + cls.content_scenarios) + + @mock.patch.object(generator, '_get_raw_opts_loaders') + def test_generate(self, raw_opts_loader): + generator.register_cli_opts(self.conf) + namespaces = [i[0] for i in self.opts] + self.config(namespace=namespaces, format_='yaml') + + # We have a static data structure matching what should be + # returned by _list_opts() but we're mocking out a lower level + # function that needs to return a namespace and a callable to + # return options from that namespace. We have to pass opts to + # the lambda to cache a reference to the name because the list + # comprehension changes the thing pointed to by the name each + # time through the loop. + raw_opts_loader.return_value = [ + (ns, lambda opts=opts: opts) + for ns, opts in self.opts + ] + test_groups = generator._get_groups( + generator._list_opts(self.conf.namespace)) + self.assertEqual(self.expected, + generator._generate_machine_readable_data(test_groups, + self.conf)) + + class IgnoreDoublesTestCase(base.BaseTestCase): opts = [cfg.StrOpt('foo', help='foo option'), @@ -1377,3 +1636,4 @@ class AdvancedOptionsTestCase(base.BaseTestCase): GeneratorTestCase.generate_scenarios() +MachineReadableGeneratorTestCase.generate_scenarios() diff --git a/releasenotes/notes/machine-readable-sample-config-e8f8ba43ababcf99.yaml b/releasenotes/notes/machine-readable-sample-config-e8f8ba43ababcf99.yaml new file mode 100644 index 0000000..179027b --- /dev/null +++ b/releasenotes/notes/machine-readable-sample-config-e8f8ba43ababcf99.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + The sample config generator can now generate machine-readable formats of + the sample config data. This can be consumed by deployment tools to + automatically generate configuration files that contain all of the + information in the traditional sample configs.