diff --git a/oslo_config/cfg.py b/oslo_config/cfg.py index b067c2d..ee9b5d8 100644 --- a/oslo_config/cfg.py +++ b/oslo_config/cfg.py @@ -231,6 +231,46 @@ group name:: --rabbit-host localhost --rabbit-port 9999 +Dynamic Groups +-------------- + +Groups can be registered dynamically by application code. This +introduces a challenge for the sample generator, discovery mechanisms, +and validation tools, since they do not know in advance the names of +all of the groups. The ``dynamic_group_owner`` parameter to the +constructor specifies the full name of an option registered in another +group that controls repeated instances of a dynamic group. This option +is usually a MultiStrOpt. + +For example, Cinder supports multiple storage backend devices and +services. To configure Cinder to communicate with multiple backends, +the ``enabled_backends`` option is set to the list of names of +backends. Each backend group includes the options for communicating +with that device or service. + +Driver Groups +------------- + +Groups can have dynamic sets of options, usually based on a driver +that has unique requirements. This works at runtime because the code +registers options before it uses them, but it introduces a challenge +for the sample generator, discovery mechanisms, and validation tools +because they do not know in advance the correct options for a group. + +To address this issue, the driver option for a group can be named +using the ``driver_option`` parameter. Each driver option should +define its own discovery entry point namespace to return the set of +options for that driver, named using the prefix +``"oslo.config.opts."`` followed by the driver option name. + +In the Cinder case described above, a ``volume_backend_name`` option +is part of the static definition of the group, so ``driver_option`` +should be set to ``"volume_backend_name"``. And plugins should be +registered under ``"oslo.config.opts.volume_backend_name"`` using the +same names as the main plugin registered with +``"oslo.config.opts"``. The drivers residing within the Cinder code +base have an entry point named ``"cinder"`` registered. + Accessing Option Values In Your Code ------------------------------------ @@ -1709,18 +1749,51 @@ class OptGroup(object): the group description as displayed in --help :param name: the group name + :type name: str :param title: the group title for --help + :type title: str :param help: the group description for --help + :type help: str + :param dynamic_group_owner: The name of the option that controls + repeated instances of this group. + :type dynamic_group_owner: str + :param driver_option: The name of the option within the group that + controls which driver will register options. + :type driver_option: str + """ - def __init__(self, name, title=None, help=None): + def __init__(self, name, title=None, help=None, + dynamic_group_owner='', + driver_option=''): """Constructs an OptGroup object.""" self.name = name self.title = "%s options" % name if title is None else title self.help = help + self.dynamic_group_owner = dynamic_group_owner + self.driver_option = driver_option self._opts = {} # dict of dicts of (opt:, override:, default:) self._argparse_group = None + self._driver_opts = {} # populated by the config generator + + def _save_driver_opts(self, opts): + """Save known driver opts. + + :param opts: mapping between driver name and list of opts + :type opts: dict + + """ + self._driver_opts.update(opts) + + def _get_generator_data(self): + "Return a dict with data for the sample generator." + return { + 'help': self.help or '', + 'dynamic_group_owner': self.dynamic_group_owner, + 'driver_option': self.driver_option, + 'driver_opts': self._driver_opts, + } def _register_opt(self, opt, cli=False): """Add an opt to this group. diff --git a/oslo_config/generator.py b/oslo_config/generator.py index bbf1737..ba3d3cc 100644 --- a/oslo_config/generator.py +++ b/oslo_config/generator.py @@ -410,6 +410,33 @@ def _get_raw_opts_loaders(namespaces): return [(e.name, e.plugin) for e in mgr] +def _get_driver_opts_loaders(namespaces, driver_option_name): + mgr = stevedore.named.NamedExtensionManager( + namespace='oslo.config.opts.' + driver_option_name, + names=namespaces, + on_load_failure_callback=on_load_failure_callback, + invoke_on_load=False) + return [(e.name, e.plugin) for e in mgr] + + +def _get_driver_opts(driver_option_name, namespaces): + """List the options available from plugins for drivers based on the option. + + :param driver_option_name: The name of the option controlling the + driver options. + :param namespaces: a list of namespaces registered under + 'oslo.config.opts.' + driver_option_name + :returns: a dict mapping driver name to option list + + """ + all_opts = {} + loaders = _get_driver_opts_loaders(namespaces, driver_option_name) + for plugin_name, loader in loaders: + for driver_name, option_list in loader().items(): + all_opts.setdefault(driver_name, []).extend(option_list) + return all_opts + + def _get_opt_default_updaters(namespaces): mgr = stevedore.named.NamedExtensionManager( 'oslo.config.opts.defaults', @@ -441,11 +468,41 @@ def _list_opts(namespaces): _update_defaults(namespaces) # Ask for the option definitions. At this point any global default # changes made by the updaters should be in effect. - opts = [ - (namespace, loader()) - for namespace, loader in loaders - ] - return _cleanup_opts(opts) + response = [] + for namespace, loader in loaders: + # The loaders return iterables for the group opts, and we need + # to extend them, so build a list. + namespace_values = [] + # Look through the groups and find any that need drivers so we + # can load those extra options. + for group, group_opts in loader(): + # group_opts is an iterable but we are going to extend it + # so convert it to a list. + group_opts = list(group_opts) + if isinstance(group, cfg.OptGroup): + if group.driver_option: + # Load the options for all of the known drivers. + driver_opts = _get_driver_opts( + group.driver_option, + namespaces, + ) + # Save the list of names of options for each + # driver in the group for use later. Add the + # options to the group_opts list so they are + # processed along with the static options in that + # group. + driver_opt_names = {} + for driver_name, opts in sorted(driver_opts.items()): + # Multiple plugins may add values to the same + # driver name, so combine the lists we do + # find. + driver_opt_names.setdefault(driver_name, []).extend( + o.name for o in opts) + group_opts.extend(opts) + group._save_driver_opts(driver_opt_names) + namespace_values.append((group, group_opts)) + response.append((namespace, namespace_values)) + return _cleanup_opts(response) def on_load_failure_callback(*args, **kwargs): @@ -565,14 +622,22 @@ def _generate_machine_readable_data(groups, conf): 'generator_options': {}} # See _get_groups for details on the structure of group_data for group_name, group_data in groups.items(): - output_data['options'][group_name] = {'opts': [], 'help': ''} + output_group = {'opts': [], 'help': ''} + output_data['options'][group_name] = output_group for namespace in group_data['namespaces']: for opt in namespace[1]: if group_data['object']: - output_group = output_data['options'][group_name] - output_group['help'] = group_data['object'].help + output_group.update( + group_data['object']._get_generator_data() + ) + else: + output_group.update({ + 'dynamic_group_owner': '', + 'driver_option': '', + 'driver_opts': {}, + }) entry = _build_entry(opt, group_name, namespace[0], conf) - output_data['options'][group_name]['opts'].append(entry) + output_group['opts'].append(entry) # Need copies of the opts because we modify them for deprecated_opt in copy.deepcopy(entry['deprecated_opts']): group = deprecated_opt.pop('group') @@ -581,6 +646,16 @@ def _generate_machine_readable_data(groups, conf): deprecated_opt['replacement_name'] = entry['name'] deprecated_opt['replacement_group'] = group_name deprecated_options[group].append(deprecated_opt) + # Build the list of options in the group that are not tied to + # a driver. + non_driver_opt_names = [ + o['name'] + for o in output_group['opts'] + if not any(o['name'] in output_group['driver_opts'][d] + for d in output_group['driver_opts']) + ] + output_group['standard_opts'] = non_driver_opt_names + output_data['generator_options'] = dict(conf) return output_data @@ -605,7 +680,7 @@ def _output_machine_readable(groups, output_file, conf): output_file.write('\n') -def generate(conf): +def generate(conf, output_file=None): """Generate a sample config file. List all of the options available via the namespaces specified in the given @@ -615,8 +690,9 @@ def generate(conf): """ conf.register_opts(_generator_opts) - output_file = (open(conf.output_file, 'w') - if conf.output_file else sys.stdout) + if output_file is None: + output_file = (open(conf.output_file, 'w') + if conf.output_file else sys.stdout) groups = _get_groups(_list_opts(conf.namespace)) diff --git a/oslo_config/tests/test_generator.py b/oslo_config/tests/test_generator.py index 8b075e9..8c7651e 100644 --- a/oslo_config/tests/test_generator.py +++ b/oslo_config/tests/test_generator.py @@ -26,6 +26,9 @@ from oslo_config import fixture as config_fixture from oslo_config import generator from oslo_config import types +import yaml + + load_tests = testscenarios.load_tests_apply_scenarios @@ -954,6 +957,80 @@ class GeneratorTestCase(base.BaseTestCase): self.assertFalse(mock_log.warning.called) +class DriverOptionTestCase(base.BaseTestCase): + + def setUp(self): + super(DriverOptionTestCase, self).setUp() + + self.conf = cfg.ConfigOpts() + self.config_fixture = config_fixture.Config(self.conf) + self.config = self.config_fixture.config + self.useFixture(self.config_fixture) + + @mock.patch.object(generator, '_get_driver_opts_loaders') + @mock.patch.object(generator, '_get_raw_opts_loaders') + @mock.patch.object(generator, 'LOG') + def test_driver_option(self, mock_log, raw_opts_loader, + driver_opts_loader): + group = cfg.OptGroup( + name='test_group', + title='Test Group', + driver_option='foo', + ) + regular_opts = [ + cfg.MultiStrOpt('foo', help='foo option'), + cfg.StrOpt('bar', help='bar option'), + ] + driver_opts = { + 'd1': [ + cfg.StrOpt('d1-foo', help='foo option'), + ], + 'd2': [ + cfg.StrOpt('d2-foo', help='foo option'), + ], + } + + # We have a static data structure matching what should be + # returned by _list_opts() but we're mocking out a lower level + # function that needs to return a namespace and a callable to + # return options from that namespace. We have to pass opts to + # the lambda to cache a reference to the name because the list + # comprehension changes the thing pointed to by the name each + # time through the loop. + raw_opts_loader.return_value = [ + ('testing', lambda: [(group, regular_opts)]), + ] + driver_opts_loader.return_value = [ + ('testing', lambda: driver_opts), + ] + + # Initialize the generator to produce YAML output to a buffer. + generator.register_cli_opts(self.conf) + self.config(namespace=['test_generator'], format_='yaml') + stdout = moves.StringIO() + + # Generate the output and parse it back to a data structure. + generator.generate(self.conf, output_file=stdout) + body = stdout.getvalue() + actual = yaml.safe_load(body) + + test_section = actual['options']['test_group'] + + self.assertEqual('foo', test_section['driver_option']) + found_option_names = [ + o['name'] + for o in test_section['opts'] + ] + self.assertEqual( + ['foo', 'bar', 'd1-foo', 'd2-foo'], + found_option_names + ) + self.assertEqual( + {'d1': ['d1-foo'], 'd2': ['d2-foo']}, + test_section['driver_opts'], + ) + + GENERATOR_OPTS = {'format_': 'yaml', 'minimal': False, 'namespace': ['test'], @@ -972,7 +1049,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': '', + 'standard_opts': ['foo'], 'opts': [{'advanced': False, 'choices': [], 'default': None, @@ -1000,7 +1081,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': '', + 'standard_opts': ['long_help'], 'opts': [{'advanced': False, 'choices': [], 'default': None, @@ -1028,7 +1113,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': '', + 'standard_opts': ['long_help_pre'], 'opts': [{'advanced': False, 'choices': [], 'default': None, @@ -1061,7 +1150,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': '', + 'standard_opts': ['foo-bar'], 'opts': [{ 'advanced': False, 'choices': [], @@ -1092,7 +1185,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': '', + 'standard_opts': ['choices_opt'], 'opts': [{'advanced': False, 'choices': (None, '', 'a', 'b', 'c'), 'default': 'a', @@ -1120,7 +1217,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': '', + 'standard_opts': ['int_opt'], 'opts': [{'advanced': False, 'choices': [], 'default': 10, @@ -1148,11 +1249,19 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase): 'generator_options': GENERATOR_OPTS, 'options': { 'DEFAULT': { + # 'driver_option': '', + # 'driver_opts': [], + # 'dynamic_group_owner': '', 'help': '', + 'standard_opts': [], 'opts': [] }, 'group1': { + 'driver_option': '', + 'driver_opts': {}, + 'dynamic_group_owner': '', 'help': all_groups['group1'].help, + 'standard_opts': ['foo'], 'opts': [{'advanced': False, 'choices': [], 'default': None,