From f74d47c8d008688873875243e9a73f4e070006db Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 14 Jun 2017 13:42:27 -0400 Subject: [PATCH] add to group data model to for generator Address some of the challenges the sample config generator has when dealing with dynamically created groups and with options that may change based on a driver or other plugin selection. Change-Id: I66b195835a7db3e32b16ddac2166782ff8592806 Signed-off-by: Doug Hellmann --- oslo_config/cfg.py | 75 ++++++++++++++++++- oslo_config/generator.py | 100 ++++++++++++++++++++++--- oslo_config/tests/test_generator.py | 109 ++++++++++++++++++++++++++++ 3 files changed, 271 insertions(+), 13 deletions(-) diff --git a/oslo_config/cfg.py b/oslo_config/cfg.py index 94d487e..9d5c96a 100644 --- a/oslo_config/cfg.py +++ b/oslo_config/cfg.py @@ -231,6 +231,46 @@ group name:: --rabbit-host localhost --rabbit-port 9999 +Dynamic Groups +-------------- + +Groups can be registered dynamically by application code. This +introduces a challenge for the sample generator, discovery mechanisms, +and validation tools, since they do not know in advance the names of +all of the groups. The ``dynamic_group_owner`` parameter to the +constructor specifies the full name of an option registered in another +group that controls repeated instances of a dynamic group. This option +is usually a MultiStrOpt. + +For example, Cinder supports multiple storage backend devices and +services. To configure Cinder to communicate with multiple backends, +the ``enabled_backends`` option is set to the list of names of +backends. Each backend group includes the options for communicating +with that device or service. + +Driver Groups +------------- + +Groups can have dynamic sets of options, usually based on a driver +that has unique requirements. This works at runtime because the code +registers options before it uses them, but it introduces a challenge +for the sample generator, discovery mechanisms, and validation tools +because they do not know in advance the correct options for a group. + +To address this issue, the driver option for a group can be named +using the ``driver_option`` parameter. Each driver option should +define its own discovery entry point namespace to return the set of +options for that driver, named using the prefix +``"oslo.config.opts."`` followed by the driver option name. + +In the Cinder case described above, a ``volume_backend_name`` option +is part of the static definition of the group, so ``driver_option`` +should be set to ``"volume_backend_name"``. And plugins should be +registered under ``"oslo.config.opts.volume_backend_name"`` using the +same names as the main plugin registered with +``"oslo.config.opts"``. The drivers residing within the Cinder code +base have an entry point named ``"cinder"`` registered. + Accessing Option Values In Your Code ------------------------------------ @@ -1694,18 +1734,51 @@ class OptGroup(object): the group description as displayed in --help :param name: the group name + :type name: str :param title: the group title for --help + :type title: str :param help: the group description for --help + :type help: str + :param dynamic_group_owner: The name of the option that controls + repeated instances of this group. + :type dynamic_group_owner: str + :param driver_option: The name of the option within the group that + controls which driver will register options. + :type driver_option: str + """ - def __init__(self, name, title=None, help=None): + def __init__(self, name, title=None, help=None, + dynamic_group_owner='', + driver_option=''): """Constructs an OptGroup object.""" self.name = name self.title = "%s options" % name if title is None else title self.help = help + self.dynamic_group_owner = dynamic_group_owner + self.driver_option = driver_option self._opts = {} # dict of dicts of (opt:, override:, default:) self._argparse_group = None + self._driver_opts = {} # populated by the config generator + + def _save_driver_opts(self, opts): + """Save known driver opts. + + :param opts: mapping between driver name and list of opts + :type opts: dict + + """ + self._driver_opts.update(opts) + + def _get_generator_data(self): + "Return a dict with data for the sample generator." + return { + 'help': self.help or '', + 'dynamic_group_owner': self.dynamic_group_owner, + 'driver_option': self.driver_option, + 'driver_opts': self._driver_opts, + } def _register_opt(self, opt, cli=False): """Add an opt to this group. diff --git a/oslo_config/generator.py b/oslo_config/generator.py index bbf1737..ba3d3cc 100644 --- a/oslo_config/generator.py +++ b/oslo_config/generator.py @@ -410,6 +410,33 @@ def _get_raw_opts_loaders(namespaces): return [(e.name, e.plugin) for e in mgr] +def _get_driver_opts_loaders(namespaces, driver_option_name): + mgr = stevedore.named.NamedExtensionManager( + namespace='oslo.config.opts.' + driver_option_name, + names=namespaces, + on_load_failure_callback=on_load_failure_callback, + invoke_on_load=False) + return [(e.name, e.plugin) for e in mgr] + + +def _get_driver_opts(driver_option_name, namespaces): + """List the options available from plugins for drivers based on the option. + + :param driver_option_name: The name of the option controlling the + driver options. + :param namespaces: a list of namespaces registered under + 'oslo.config.opts.' + driver_option_name + :returns: a dict mapping driver name to option list + + """ + all_opts = {} + loaders = _get_driver_opts_loaders(namespaces, driver_option_name) + for plugin_name, loader in loaders: + for driver_name, option_list in loader().items(): + all_opts.setdefault(driver_name, []).extend(option_list) + return all_opts + + def _get_opt_default_updaters(namespaces): mgr = stevedore.named.NamedExtensionManager( 'oslo.config.opts.defaults', @@ -441,11 +468,41 @@ def _list_opts(namespaces): _update_defaults(namespaces) # Ask for the option definitions. At this point any global default # changes made by the updaters should be in effect. - opts = [ - (namespace, loader()) - for namespace, loader in loaders - ] - return _cleanup_opts(opts) + response = [] + for namespace, loader in loaders: + # The loaders return iterables for the group opts, and we need + # to extend them, so build a list. + namespace_values = [] + # Look through the groups and find any that need drivers so we + # can load those extra options. + for group, group_opts in loader(): + # group_opts is an iterable but we are going to extend it + # so convert it to a list. + group_opts = list(group_opts) + if isinstance(group, cfg.OptGroup): + if group.driver_option: + # Load the options for all of the known drivers. + driver_opts = _get_driver_opts( + group.driver_option, + namespaces, + ) + # Save the list of names of options for each + # driver in the group for use later. Add the + # options to the group_opts list so they are + # processed along with the static options in that + # group. + driver_opt_names = {} + for driver_name, opts in sorted(driver_opts.items()): + # Multiple plugins may add values to the same + # driver name, so combine the lists we do + # find. + driver_opt_names.setdefault(driver_name, []).extend( + o.name for o in opts) + group_opts.extend(opts) + group._save_driver_opts(driver_opt_names) + namespace_values.append((group, group_opts)) + response.append((namespace, namespace_values)) + return _cleanup_opts(response) def on_load_failure_callback(*args, **kwargs): @@ -565,14 +622,22 @@ def _generate_machine_readable_data(groups, conf): 'generator_options': {}} # See _get_groups for details on the structure of group_data for group_name, group_data in groups.items(): - output_data['options'][group_name] = {'opts': [], 'help': ''} + output_group = {'opts': [], 'help': ''} + output_data['options'][group_name] = output_group for namespace in group_data['namespaces']: for opt in namespace[1]: if group_data['object']: - output_group = output_data['options'][group_name] - output_group['help'] = group_data['object'].help + output_group.update( + group_data['object']._get_generator_data() + ) + else: + output_group.update({ + 'dynamic_group_owner': '', + 'driver_option': '', + 'driver_opts': {}, + }) entry = _build_entry(opt, group_name, namespace[0], conf) - output_data['options'][group_name]['opts'].append(entry) + output_group['opts'].append(entry) # Need copies of the opts because we modify them for deprecated_opt in copy.deepcopy(entry['deprecated_opts']): group = deprecated_opt.pop('group') @@ -581,6 +646,16 @@ def _generate_machine_readable_data(groups, conf): deprecated_opt['replacement_name'] = entry['name'] deprecated_opt['replacement_group'] = group_name deprecated_options[group].append(deprecated_opt) + # Build the list of options in the group that are not tied to + # a driver. + non_driver_opt_names = [ + o['name'] + for o in output_group['opts'] + if not any(o['name'] in output_group['driver_opts'][d] + for d in output_group['driver_opts']) + ] + output_group['standard_opts'] = non_driver_opt_names + output_data['generator_options'] = dict(conf) return output_data @@ -605,7 +680,7 @@ def _output_machine_readable(groups, output_file, conf): output_file.write('\n') -def generate(conf): +def generate(conf, output_file=None): """Generate a sample config file. List all of the options available via the namespaces specified in the given @@ -615,8 +690,9 @@ def generate(conf): """ conf.register_opts(_generator_opts) - output_file = (open(conf.output_file, 'w') - if conf.output_file else sys.stdout) + if output_file is None: + output_file = (open(conf.output_file, 'w') + if conf.output_file else sys.stdout) groups = _get_groups(_list_opts(conf.namespace)) diff --git a/oslo_config/tests/test_generator.py b/oslo_config/tests/test_generator.py index 8b075e9..8c7651e 100644 --- a/oslo_config/tests/test_generator.py +++ b/oslo_config/tests/test_generator.py @@ -26,6 +26,9 @@ from oslo_config import fixture as config_fixture from oslo_config import generator from oslo_config import types +import yaml + + load_tests = testscenarios.load_tests_apply_scenarios @@ -954,6 +957,80 @@ class GeneratorTestCase(base.BaseTestCase): self.assertFalse(mock_log.warning.called) +class DriverOptionTestCase(base.BaseTestCase): + + def setUp(self): + super(DriverOptionTestCase, self).setUp() + + self.conf = cfg.ConfigOpts() + self.config_fixture = config_fixture.Config(self.conf) + self.config = self.config_fixture.config + self.useFixture(self.config_fixture) + + @mock.patch.object(generator, '_get_driver_opts_loaders') + @mock.patch.object(generator, '_get_raw_opts_loaders') + @mock.patch.object(generator, 'LOG') + def test_driver_option(self, mock_log, raw_opts_loader, + driver_opts_loader): + group = cfg.OptGroup( + name='test_group', + title='Test Group', + driver_option='foo', + ) + regular_opts = [ + cfg.MultiStrOpt('foo', help='foo option'), + cfg.StrOpt('bar', help='bar option'), + ] + driver_opts = { + 'd1': [ + cfg.StrOpt('d1-foo', help='foo option'), + ], + 'd2': [ + cfg.StrOpt('d2-foo', help='foo option'), + ], + } + + # We have a static data structure matching what should be + # returned by _list_opts() but we're mocking out a lower level + # function that needs to return a namespace and a callable to + # return options from that namespace. We have to pass opts to + # the lambda to cache a reference to the name because the list + # comprehension changes the thing pointed to by the name each + # time through the loop. + raw_opts_loader.return_value = [ + ('testing', lambda: [(group, regular_opts)]), + ] + driver_opts_loader.return_value = [ + ('testing', lambda: driver_opts), + ] + + # Initialize the generator to produce YAML output to a buffer. + generator.register_cli_opts(self.conf) + self.config(namespace=['test_generator'], format_='yaml') + stdout = moves.StringIO() + + # Generate the output and parse it back to a data structure. + generator.generate(self.conf, output_file=stdout) + body = stdout.getvalue() + actual = yaml.safe_load(body) + + test_section = actual['options']['test_group'] + + self.assertEqual('foo', test_section['driver_option']) + found_option_names = [ + o['name'] + for o in test_section['opts'] + ] + self.assertEqual( + ['foo', 'bar', 'd1-foo', 'd2-foo'], + found_option_names + ) + self.assertEqual( + {'d1': ['d1-foo'], 'd2': ['d2-foo']}, + test_section['driver_opts'], + ) + + GENERATOR_OPTS = {'format_': 'yaml', 'minimal': False, 'namespace': ['test'], @@ -972,7 +1049,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': '', + 'standard_opts': ['foo'], 'opts': [{'advanced': False, 'choices': [], 'default': None, @@ -1000,7 +1081,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': '', + 'standard_opts': ['long_help'], 'opts': [{'advanced': False, 'choices': [], 'default': None, @@ -1028,7 +1113,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': '', + 'standard_opts': ['long_help_pre'], 'opts': [{'advanced': False, 'choices': [], 'default': None, @@ -1061,7 +1150,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': '', + 'standard_opts': ['foo-bar'], 'opts': [{ 'advanced': False, 'choices': [], @@ -1092,7 +1185,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': '', + 'standard_opts': ['choices_opt'], 'opts': [{'advanced': False, 'choices': (None, '', 'a', 'b', 'c'), 'default': 'a', @@ -1120,7 +1217,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': '', + 'standard_opts': ['int_opt'], 'opts': [{'advanced': False, 'choices': [], 'default': 10, @@ -1148,11 +1249,19 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + # 'driver_option': '', + # 'driver_opts': [], + # 'dynamic_group_owner': '', 'help': '', + 'standard_opts': [], 'opts': [] }, 'group1': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': all_groups['group1'].help, + 'standard_opts': ['foo'], 'opts': [{'advanced': False, 'choices': [], 'default': None,