openstack
/
deb-python-oslo.config
Code Issues Proposed changes

add to group data model to for generator 

Address some of the challenges the sample config generator has when
dealing with dynamically created groups and with options that may change
based on a driver or other plugin selection.

Change-Id: I66b195835a7db3e32b16ddac2166782ff8592806
Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann 2017-06-14 13:42:27 -04:00
parent 9825b3746e
commit f74d47c8d0
3 changed files with 271 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
oslo_config/cfg.py
View File

@ -231,6 +231,46 @@ group name::
--rabbit-host localhost --rabbit-port 9999
Dynamic Groups
--------------
Groups can be registered dynamically by application code. This
introduces a challenge for the sample generator, discovery mechanisms,
and validation tools, since they do not know in advance the names of
all of the groups. The ``dynamic_group_owner`` parameter to the
constructor specifies the full name of an option registered in another
group that controls repeated instances of a dynamic group. This option
is usually a MultiStrOpt.
For example, Cinder supports multiple storage backend devices and
services. To configure Cinder to communicate with multiple backends,
the ``enabled_backends`` option is set to the list of names of
backends. Each backend group includes the options for communicating
with that device or service.
Driver Groups
-------------
Groups can have dynamic sets of options, usually based on a driver
that has unique requirements. This works at runtime because the code
registers options before it uses them, but it introduces a challenge
for the sample generator, discovery mechanisms, and validation tools
because they do not know in advance the correct options for a group.
To address this issue, the driver option for a group can be named
using the ``driver_option`` parameter. Each driver option should
define its own discovery entry point namespace to return the set of
options for that driver, named using the prefix
``"oslo.config.opts."`` followed by the driver option name.
In the Cinder case described above, a ``volume_backend_name`` option
is part of the static definition of the group, so ``driver_option``
should be set to ``"volume_backend_name"``. And plugins should be
registered under ``"oslo.config.opts.volume_backend_name"`` using the
same names as the main plugin registered with
``"oslo.config.opts"``. The drivers residing within the Cinder code
base have an entry point named ``"cinder"`` registered.
Accessing Option Values In Your Code
------------------------------------
@ -1694,18 +1734,51 @@ class OptGroup(object):
the group description as displayed in --help
:param name: the group name
:type name: str
:param title: the group title for --help
:type title: str
:param help: the group description for --help
:type help: str
:param dynamic_group_owner: The name of the option that controls
repeated instances of this group.
:type dynamic_group_owner: str
:param driver_option: The name of the option within the group that
controls which driver will register options.
:type driver_option: str
"""
def __init__(self, name, title=None, help=None):
def __init__(self, name, title=None, help=None,
dynamic_group_owner='',
driver_option=''):
"""Constructs an OptGroup object."""
self.name = name
self.title = "%s options" % name if title is None else title
self.help = help
self.dynamic_group_owner = dynamic_group_owner
self.driver_option = driver_option
self._opts = {} # dict of dicts of (opt:, override:, default:)
self._argparse_group = None
self._driver_opts = {} # populated by the config generator
def _save_driver_opts(self, opts):
"""Save known driver opts.
:param opts: mapping between driver name and list of opts
:type opts: dict
"""
self._driver_opts.update(opts)
def _get_generator_data(self):
"Return a dict with data for the sample generator."
return {
'help': self.help or '',
'dynamic_group_owner': self.dynamic_group_owner,
'driver_option': self.driver_option,
'driver_opts': self._driver_opts,
}
def _register_opt(self, opt, cli=False):
"""Add an opt to this group.

100
oslo_config/generator.py
View File

@ -410,6 +410,33 @@ def _get_raw_opts_loaders(namespaces):
return [(e.name, e.plugin) for e in mgr]
def _get_driver_opts_loaders(namespaces, driver_option_name):
mgr = stevedore.named.NamedExtensionManager(
namespace='oslo.config.opts.' + driver_option_name,
names=namespaces,
on_load_failure_callback=on_load_failure_callback,
invoke_on_load=False)
return [(e.name, e.plugin) for e in mgr]
def _get_driver_opts(driver_option_name, namespaces):
"""List the options available from plugins for drivers based on the option.
:param driver_option_name: The name of the option controlling the
driver options.
:param namespaces: a list of namespaces registered under
'oslo.config.opts.' + driver_option_name
:returns: a dict mapping driver name to option list
"""
all_opts = {}
loaders = _get_driver_opts_loaders(namespaces, driver_option_name)
for plugin_name, loader in loaders:
for driver_name, option_list in loader().items():
all_opts.setdefault(driver_name, []).extend(option_list)
return all_opts
def _get_opt_default_updaters(namespaces):
mgr = stevedore.named.NamedExtensionManager(
'oslo.config.opts.defaults',
@ -441,11 +468,41 @@ def _list_opts(namespaces):
_update_defaults(namespaces)
# Ask for the option definitions. At this point any global default
# changes made by the updaters should be in effect.
opts = [
(namespace, loader())
for namespace, loader in loaders
]
return _cleanup_opts(opts)
response = []
for namespace, loader in loaders:
# The loaders return iterables for the group opts, and we need
# to extend them, so build a list.
namespace_values = []
# Look through the groups and find any that need drivers so we
# can load those extra options.
for group, group_opts in loader():
# group_opts is an iterable but we are going to extend it
# so convert it to a list.
group_opts = list(group_opts)
if isinstance(group, cfg.OptGroup):
if group.driver_option:
# Load the options for all of the known drivers.
driver_opts = _get_driver_opts(
group.driver_option,
namespaces,
)
# Save the list of names of options for each
# driver in the group for use later. Add the
# options to the group_opts list so they are
# processed along with the static options in that
# group.
driver_opt_names = {}
for driver_name, opts in sorted(driver_opts.items()):
# Multiple plugins may add values to the same
# driver name, so combine the lists we do
# find.
driver_opt_names.setdefault(driver_name, []).extend(
o.name for o in opts)
group_opts.extend(opts)
group._save_driver_opts(driver_opt_names)
namespace_values.append((group, group_opts))
response.append((namespace, namespace_values))
return _cleanup_opts(response)
def on_load_failure_callback(*args, **kwargs):
@ -565,14 +622,22 @@ def _generate_machine_readable_data(groups, conf):
'generator_options': {}}
# See _get_groups for details on the structure of group_data
for group_name, group_data in groups.items():
output_data['options'][group_name] = {'opts': [], 'help': ''}
output_group = {'opts': [], 'help': ''}
output_data['options'][group_name] = output_group
for namespace in group_data['namespaces']:
for opt in namespace[1]:
if group_data['object']:
output_group = output_data['options'][group_name]
output_group['help'] = group_data['object'].help
output_group.update(
group_data['object']._get_generator_data()
)
else:
output_group.update({
'dynamic_group_owner': '',
'driver_option': '',
'driver_opts': {},
})
entry = _build_entry(opt, group_name, namespace[0], conf)
output_data['options'][group_name]['opts'].append(entry)
output_group['opts'].append(entry)
# Need copies of the opts because we modify them
for deprecated_opt in copy.deepcopy(entry['deprecated_opts']):
group = deprecated_opt.pop('group')
@ -581,6 +646,16 @@ def _generate_machine_readable_data(groups, conf):
deprecated_opt['replacement_name'] = entry['name']
deprecated_opt['replacement_group'] = group_name
deprecated_options[group].append(deprecated_opt)
# Build the list of options in the group that are not tied to
# a driver.
non_driver_opt_names = [
o['name']
for o in output_group['opts']
if not any(o['name'] in output_group['driver_opts'][d]
for d in output_group['driver_opts'])
]
output_group['standard_opts'] = non_driver_opt_names
output_data['generator_options'] = dict(conf)
return output_data
@ -605,7 +680,7 @@ def _output_machine_readable(groups, output_file, conf):
output_file.write('\n')
def generate(conf):
def generate(conf, output_file=None):
"""Generate a sample config file.
List all of the options available via the namespaces specified in the given
@ -615,8 +690,9 @@ def generate(conf):
"""
conf.register_opts(_generator_opts)
output_file = (open(conf.output_file, 'w')
if conf.output_file else sys.stdout)
if output_file is None:
output_file = (open(conf.output_file, 'w')
if conf.output_file else sys.stdout)
groups = _get_groups(_list_opts(conf.namespace))

109
oslo_config/tests/test_generator.py
View File

@ -26,6 +26,9 @@ from oslo_config import fixture as config_fixture
from oslo_config import generator
from oslo_config import types
import yaml
load_tests = testscenarios.load_tests_apply_scenarios
@ -954,6 +957,80 @@ class GeneratorTestCase(base.BaseTestCase):
self.assertFalse(mock_log.warning.called)
class DriverOptionTestCase(base.BaseTestCase):
def setUp(self):
super(DriverOptionTestCase, self).setUp()
self.conf = cfg.ConfigOpts()
self.config_fixture = config_fixture.Config(self.conf)
self.config = self.config_fixture.config
self.useFixture(self.config_fixture)
@mock.patch.object(generator, '_get_driver_opts_loaders')
@mock.patch.object(generator, '_get_raw_opts_loaders')
@mock.patch.object(generator, 'LOG')
def test_driver_option(self, mock_log, raw_opts_loader,
driver_opts_loader):
group = cfg.OptGroup(
name='test_group',
title='Test Group',
driver_option='foo',
)
regular_opts = [
cfg.MultiStrOpt('foo', help='foo option'),
cfg.StrOpt('bar', help='bar option'),
]
driver_opts = {
'd1': [
cfg.StrOpt('d1-foo', help='foo option'),
],
'd2': [
cfg.StrOpt('d2-foo', help='foo option'),
],
}
# We have a static data structure matching what should be
# returned by _list_opts() but we're mocking out a lower level
# function that needs to return a namespace and a callable to
# return options from that namespace. We have to pass opts to
# the lambda to cache a reference to the name because the list
# comprehension changes the thing pointed to by the name each
# time through the loop.
raw_opts_loader.return_value = [
('testing', lambda: [(group, regular_opts)]),
]
driver_opts_loader.return_value = [
('testing', lambda: driver_opts),
]
# Initialize the generator to produce YAML output to a buffer.
generator.register_cli_opts(self.conf)
self.config(namespace=['test_generator'], format_='yaml')
stdout = moves.StringIO()
# Generate the output and parse it back to a data structure.
generator.generate(self.conf, output_file=stdout)
body = stdout.getvalue()
actual = yaml.safe_load(body)
test_section = actual['options']['test_group']
self.assertEqual('foo', test_section['driver_option'])
found_option_names = [
o['name']
for o in test_section['opts']
]
self.assertEqual(
['foo', 'bar', 'd1-foo', 'd2-foo'],
found_option_names
)
self.assertEqual(
{'d1': ['d1-foo'], 'd2': ['d2-foo']},
test_section['driver_opts'],
)
GENERATOR_OPTS = {'format_': 'yaml',
'minimal': False,
'namespace': ['test'],
@ -972,7 +1049,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': '',
'standard_opts': ['foo'],
'opts': [{'advanced': False,
'choices': [],
'default': None,
@ -1000,7 +1081,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': '',
'standard_opts': ['long_help'],
'opts': [{'advanced': False,
'choices': [],
'default': None,
@ -1028,7 +1113,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': '',
'standard_opts': ['long_help_pre'],
'opts': [{'advanced': False,
'choices': [],
'default': None,
@ -1061,7 +1150,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': '',
'standard_opts': ['foo-bar'],
'opts': [{
'advanced': False,
'choices': [],
@ -1092,7 +1185,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': '',
'standard_opts': ['choices_opt'],
'opts': [{'advanced': False,
'choices': (None, '', 'a', 'b', 'c'),
'default': 'a',
@ -1120,7 +1217,11 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': '',
'standard_opts': ['int_opt'],
'opts': [{'advanced': False,
'choices': [],
'default': 10,
@ -1148,11 +1249,19 @@ class MachineReadableGeneratorTestCase(base.BaseTestCase):
'generator_options': GENERATOR_OPTS,
'options': {
'DEFAULT': {
# 'driver_option': '',
# 'driver_opts': [],
# 'dynamic_group_owner': '',
'help': '',
'standard_opts': [],
'opts': []
},
'group1': {
'driver_option': '',
'driver_opts': {},
'dynamic_group_owner': '',
'help': all_groups['group1'].help,
'standard_opts': ['foo'],
'opts': [{'advanced': False,
'choices': [],
'default': None,