Import latest openstack-common
Main change is the MultiStrOpt support which pulls in the new iniparser module. See lp#955308. Change-Id: I1358f09038113873e60343f00a95229002b58c1f
This commit is contained in:
Mark McLoughlin
parent
75a8dfef51
commit
357138dd9d
|
|
@ -17,7 +17,9 @@
|
|||
r"""
|
||||
Configuration options which may be set on the command line or in config files.
|
||||
|
||||
The schema for each option is defined using the Opt sub-classes e.g.
|
||||
The schema for each option is defined using the Opt sub-classes, e.g.:
|
||||
|
||||
::
|
||||
|
||||
common_opts = [
|
||||
cfg.StrOpt('bind_host',
|
||||
|
|
@ -28,20 +30,20 @@ The schema for each option is defined using the Opt sub-classes e.g.
|
|||
help='Port number to listen on')
|
||||
]
|
||||
|
||||
Options can be strings, integers, floats, booleans, lists or 'multi strings':
|
||||
Options can be strings, integers, floats, booleans, lists or 'multi strings'::
|
||||
|
||||
enabled_apis_opt = cfg.ListOpt('enabled_apis',
|
||||
default=['ec2', 'osapi_compute'],
|
||||
help='List of APIs to enable by default')
|
||||
|
||||
DEFAULT_EXTENSIONS = [
|
||||
'nova.api.openstack.contrib.standard_extensions'
|
||||
'nova.api.openstack.compute.contrib.standard_extensions'
|
||||
]
|
||||
osapi_compute_extension_opt = cfg.MultiStrOpt('osapi_compute_extension',
|
||||
default=DEFAULT_EXTENSIONS)
|
||||
|
||||
Option schemas are registered with with the config manager at runtime, but
|
||||
before the option is referenced:
|
||||
before the option is referenced::
|
||||
|
||||
class ExtensionManager(object):
|
||||
|
||||
|
|
@ -57,7 +59,7 @@ before the option is referenced:
|
|||
....
|
||||
|
||||
A common usage pattern is for each option schema to be defined in the module or
|
||||
class which uses the option:
|
||||
class which uses the option::
|
||||
|
||||
opts = ...
|
||||
|
||||
|
|
@ -72,7 +74,7 @@ class which uses the option:
|
|||
|
||||
An option may optionally be made available via the command line. Such options
|
||||
must registered with the config manager before the command line is parsed (for
|
||||
the purposes of --help and CLI arg validation):
|
||||
the purposes of --help and CLI arg validation)::
|
||||
|
||||
cli_opts = [
|
||||
cfg.BoolOpt('verbose',
|
||||
|
|
@ -88,7 +90,7 @@ the purposes of --help and CLI arg validation):
|
|||
def add_common_opts(conf):
|
||||
conf.register_cli_opts(cli_opts)
|
||||
|
||||
The config manager has a single CLI option defined by default, --config-file:
|
||||
The config manager has a single CLI option defined by default, --config-file::
|
||||
|
||||
class ConfigOpts(object):
|
||||
|
||||
|
|
@ -99,9 +101,9 @@ The config manager has a single CLI option defined by default, --config-file:
|
|||
...
|
||||
self.register_cli_opt(self.config_file_opt)
|
||||
|
||||
Option values are parsed from any supplied config files using SafeConfigParser.
|
||||
If none are specified, a default set is used e.g. glance-api.conf and
|
||||
glance-common.conf:
|
||||
Option values are parsed from any supplied config files using
|
||||
openstack.common.iniparser. If none are specified, a default set is used
|
||||
e.g. glance-api.conf and glance-common.conf::
|
||||
|
||||
glance-api.conf:
|
||||
[DEFAULT]
|
||||
|
|
@ -116,7 +118,7 @@ are parsed in order, with values in later files overriding those in earlier
|
|||
files.
|
||||
|
||||
The parsing of CLI args and config files is initiated by invoking the config
|
||||
manager e.g.
|
||||
manager e.g.::
|
||||
|
||||
conf = ConfigOpts()
|
||||
conf.register_opt(BoolOpt('verbose', ...))
|
||||
|
|
@ -124,10 +126,10 @@ manager e.g.
|
|||
if conf.verbose:
|
||||
...
|
||||
|
||||
Options can be registered as belonging to a group:
|
||||
Options can be registered as belonging to a group::
|
||||
|
||||
rabbit_group = cfg.OptionGroup(name='rabbit',
|
||||
title='RabbitMQ options')
|
||||
rabbit_group = cfg.OptGroup(name='rabbit',
|
||||
title='RabbitMQ options')
|
||||
|
||||
rabbit_host_opt = cfg.StrOpt('host',
|
||||
default='localhost',
|
||||
|
|
@ -143,7 +145,7 @@ Options can be registered as belonging to a group:
|
|||
conf.register_opt(rabbit_port_opt, group='rabbit')
|
||||
|
||||
If no group is specified, options belong to the 'DEFAULT' section of config
|
||||
files:
|
||||
files::
|
||||
|
||||
glance-api.conf:
|
||||
[DEFAULT]
|
||||
|
|
@ -158,13 +160,14 @@ files:
|
|||
password = guest
|
||||
virtual_host = /
|
||||
|
||||
Command-line options in a group are automatically prefixed with the group name:
|
||||
Command-line options in a group are automatically prefixed with the
|
||||
group name::
|
||||
|
||||
--rabbit-host localhost --rabbit-port 9999
|
||||
|
||||
Option values in the default group are referenced as attributes/properties on
|
||||
the config manager; groups are also attributes on the config manager, with
|
||||
attributes for each of the options associated with the group:
|
||||
attributes for each of the options associated with the group::
|
||||
|
||||
server.start(app, conf.bind_port, conf.bind_host, conf)
|
||||
|
||||
|
|
@ -173,7 +176,7 @@ attributes for each of the options associated with the group:
|
|||
port=conf.rabbit.port,
|
||||
...)
|
||||
|
||||
Option values may reference other values using PEP 292 string substitution:
|
||||
Option values may reference other values using PEP 292 string substitution::
|
||||
|
||||
opts = [
|
||||
cfg.StrOpt('state_path',
|
||||
|
|
@ -191,29 +194,40 @@ Note that interpolation can be avoided by using '$$'.
|
|||
|
||||
For command line utilities that dispatch to other command line utilities, the
|
||||
disable_interspersed_args() method is available. If this this method is called,
|
||||
then parsing e.g.
|
||||
then parsing e.g.::
|
||||
|
||||
script --verbose cmd --debug /tmp/mything
|
||||
|
||||
will no longer return:
|
||||
will no longer return::
|
||||
|
||||
['cmd', '/tmp/mything']
|
||||
|
||||
as the leftover arguments, but will instead return:
|
||||
as the leftover arguments, but will instead return::
|
||||
|
||||
['cmd', '--debug', '/tmp/mything']
|
||||
|
||||
i.e. argument parsing is stopped at the first non-option argument.
|
||||
|
||||
Options may be declared as secret so that their values are not leaked into
|
||||
log files:
|
||||
|
||||
opts = [
|
||||
cfg.StrOpt('s3_store_access_key', secret=True),
|
||||
cfg.StrOpt('s3_store_secret_key', secret=True),
|
||||
...
|
||||
]
|
||||
|
||||
"""
|
||||
|
||||
import collections
|
||||
import ConfigParser
|
||||
import copy
|
||||
import optparse
|
||||
import os
|
||||
import string
|
||||
import sys
|
||||
|
||||
from keystone.openstack.common import iniparser
|
||||
|
||||
|
||||
class Error(Exception):
|
||||
"""Base class for cfg exceptions."""
|
||||
|
|
@ -307,9 +321,12 @@ class ConfigFileValueError(Error):
|
|||
def find_config_files(project=None, prog=None):
|
||||
"""Return a list of default configuration files.
|
||||
|
||||
:param project: an optional project name
|
||||
:param prog: the program name, defaulting to the basename of sys.argv[0]
|
||||
|
||||
We default to two config files: [${project}.conf, ${prog}.conf]
|
||||
|
||||
And we look for those config files in the following directories:
|
||||
And we look for those config files in the following directories::
|
||||
|
||||
~/.${project}/
|
||||
~/
|
||||
|
|
@ -324,9 +341,6 @@ def find_config_files(project=None, prog=None):
|
|||
'~/.foo/bar.conf']
|
||||
|
||||
If no project name is supplied, we only look for ${prog.conf}.
|
||||
|
||||
:param project: an optional project name
|
||||
:param prog: the program name, defaulting to the basename of sys.argv[0]
|
||||
"""
|
||||
if prog is None:
|
||||
prog = os.path.basename(sys.argv[0])
|
||||
|
|
@ -395,9 +409,10 @@ class Opt(object):
|
|||
help:
|
||||
an string explaining how the options value is used
|
||||
"""
|
||||
multi = False
|
||||
|
||||
def __init__(self, name, dest=None, short=None,
|
||||
default=None, metavar=None, help=None):
|
||||
def __init__(self, name, dest=None, short=None, default=None,
|
||||
metavar=None, help=None, secret=False):
|
||||
"""Construct an Opt object.
|
||||
|
||||
The only required parameter is the option's name. However, it is
|
||||
|
|
@ -409,6 +424,7 @@ class Opt(object):
|
|||
:param default: the default value of the option
|
||||
:param metavar: the option argument to show in --help
|
||||
:param help: an explanation of how the option is used
|
||||
:param secret: true iff the value should be obfuscated in log output
|
||||
"""
|
||||
self.name = name
|
||||
if dest is None:
|
||||
|
|
@ -419,9 +435,10 @@ class Opt(object):
|
|||
self.default = default
|
||||
self.metavar = metavar
|
||||
self.help = help
|
||||
self.secret = secret
|
||||
|
||||
def _get_from_config_parser(self, cparser, section):
|
||||
"""Retrieves the option value from a ConfigParser object.
|
||||
"""Retrieves the option value from a MultiConfigParser object.
|
||||
|
||||
This is the method ConfigOpts uses to look up the option value from
|
||||
config files. Most opt types override this method in order to perform
|
||||
|
|
@ -430,7 +447,7 @@ class Opt(object):
|
|||
:param cparser: a ConfigParser object
|
||||
:param section: a section name
|
||||
"""
|
||||
return cparser.get(section, self.dest, raw=True)
|
||||
return cparser.get(section, self.dest)
|
||||
|
||||
def _add_to_cli(self, parser, group=None):
|
||||
"""Makes the option available in the command line interface.
|
||||
|
|
@ -532,9 +549,19 @@ class BoolOpt(Opt):
|
|||
1/0, yes/no, true/false or on/off.
|
||||
"""
|
||||
|
||||
_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
|
||||
'0': False, 'no': False, 'false': False, 'off': False}
|
||||
|
||||
def _get_from_config_parser(self, cparser, section):
|
||||
"""Retrieve the opt value as a boolean from ConfigParser."""
|
||||
return cparser.getboolean(section, self.dest)
|
||||
def convert_bool(v):
|
||||
value = self._boolean_states.get(v.lower())
|
||||
if value is None:
|
||||
raise ValueError('Unexpected boolean value %r' % v)
|
||||
|
||||
return value
|
||||
|
||||
return [convert_bool(v) for v in cparser.get(section, self.dest)]
|
||||
|
||||
def _add_to_cli(self, parser, group=None):
|
||||
"""Extends the base class method to add the --nooptname option."""
|
||||
|
|
@ -561,7 +588,7 @@ class IntOpt(Opt):
|
|||
|
||||
def _get_from_config_parser(self, cparser, section):
|
||||
"""Retrieve the opt value as a integer from ConfigParser."""
|
||||
return cparser.getint(section, self.dest)
|
||||
return [int(v) for v in cparser.get(section, self.dest)]
|
||||
|
||||
def _get_optparse_kwargs(self, group, **kwargs):
|
||||
"""Extends the base optparse keyword dict for integer options."""
|
||||
|
|
@ -575,7 +602,7 @@ class FloatOpt(Opt):
|
|||
|
||||
def _get_from_config_parser(self, cparser, section):
|
||||
"""Retrieve the opt value as a float from ConfigParser."""
|
||||
return cparser.getfloat(section, self.dest)
|
||||
return [float(v) for v in cparser.get(section, self.dest)]
|
||||
|
||||
def _get_optparse_kwargs(self, group, **kwargs):
|
||||
"""Extends the base optparse keyword dict for float options."""
|
||||
|
|
@ -592,7 +619,7 @@ class ListOpt(Opt):
|
|||
|
||||
def _get_from_config_parser(self, cparser, section):
|
||||
"""Retrieve the opt value as a list from ConfigParser."""
|
||||
return cparser.get(section, self.dest).split(',')
|
||||
return [v.split(',') for v in cparser.get(section, self.dest)]
|
||||
|
||||
def _get_optparse_kwargs(self, group, **kwargs):
|
||||
"""Extends the base optparse keyword dict for list options."""
|
||||
|
|
@ -614,14 +641,7 @@ class MultiStrOpt(Opt):
|
|||
Multistr opt values are string opts which may be specified multiple times.
|
||||
The opt value is a list containing all the string values specified.
|
||||
"""
|
||||
|
||||
def _get_from_config_parser(self, cparser, section):
|
||||
"""Retrieve the opt value as a multistr from ConfigParser."""
|
||||
# FIXME(markmc): values spread across the CLI and multiple
|
||||
# config files should be appended
|
||||
value = super(MultiStrOpt, self)._get_from_config_parser(cparser,
|
||||
section)
|
||||
return value if value is None else [value]
|
||||
multi = True
|
||||
|
||||
def _get_optparse_kwargs(self, group, **kwargs):
|
||||
"""Extends the base optparse keyword dict for multi str options."""
|
||||
|
|
@ -688,6 +708,69 @@ class OptGroup(object):
|
|||
return self._optparse_group
|
||||
|
||||
|
||||
class ParseError(iniparser.ParseError):
|
||||
def __init__(self, msg, lineno, line, filename):
|
||||
super(ParseError, self).__init__(msg, lineno, line)
|
||||
self.filename = filename
|
||||
|
||||
def __str__(self):
|
||||
return 'at %s:%d, %s: %r' % (self.filename, self.lineno,
|
||||
self.msg, self.line)
|
||||
|
||||
|
||||
class ConfigParser(iniparser.BaseParser):
|
||||
def __init__(self, filename, sections):
|
||||
super(ConfigParser, self).__init__()
|
||||
self.filename = filename
|
||||
self.sections = sections
|
||||
self.section = None
|
||||
|
||||
def parse(self):
|
||||
with open(self.filename) as f:
|
||||
return super(ConfigParser, self).parse(f)
|
||||
|
||||
def new_section(self, section):
|
||||
self.section = section
|
||||
self.sections.setdefault(self.section, {})
|
||||
|
||||
def assignment(self, key, value):
|
||||
if not self.section:
|
||||
raise self.error_no_section()
|
||||
|
||||
self.sections[self.section].setdefault(key, [])
|
||||
self.sections[self.section][key].append('\n'.join(value))
|
||||
|
||||
def parse_exc(self, msg, lineno, line=None):
|
||||
return ParseError(msg, lineno, line, self.filename)
|
||||
|
||||
def error_no_section(self):
|
||||
return self.parse_exc('Section must be started before assignment',
|
||||
self.lineno)
|
||||
|
||||
|
||||
class MultiConfigParser(object):
|
||||
def __init__(self):
|
||||
self.sections = {}
|
||||
|
||||
def read(self, config_files):
|
||||
read_ok = []
|
||||
|
||||
for filename in config_files:
|
||||
parser = ConfigParser(filename, self.sections)
|
||||
|
||||
try:
|
||||
parser.parse()
|
||||
except IOError:
|
||||
continue
|
||||
|
||||
read_ok.append(filename)
|
||||
|
||||
return read_ok
|
||||
|
||||
def get(self, section, name):
|
||||
return self.sections[section][name]
|
||||
|
||||
|
||||
class ConfigOpts(collections.Mapping):
|
||||
|
||||
"""
|
||||
|
|
@ -945,15 +1028,22 @@ class ConfigOpts(collections.Mapping):
|
|||
logger.log(lvl, "config files: %s", self.config_file)
|
||||
logger.log(lvl, "=" * 80)
|
||||
|
||||
def _sanitize(opt, value):
|
||||
"""Obfuscate values of options declared secret"""
|
||||
return value if not opt.secret else '*' * len(str(value))
|
||||
|
||||
for opt_name in sorted(self._opts):
|
||||
logger.log(lvl, "%-30s = %s", opt_name, getattr(self, opt_name))
|
||||
opt = self._get_opt_info(opt_name)['opt']
|
||||
logger.log(lvl, "%-30s = %s", opt_name,
|
||||
_sanitize(opt, getattr(self, opt_name)))
|
||||
|
||||
for group_name in self._groups:
|
||||
group_attr = self.GroupAttr(self, self._get_group(group_name))
|
||||
for opt_name in sorted(self._groups[group_name]._opts):
|
||||
opt = self._get_opt_info(opt_name, group_name)['opt']
|
||||
logger.log(lvl, "%-30s = %s",
|
||||
"%s.%s" % (group_name, opt_name),
|
||||
getattr(group_attr, opt_name))
|
||||
_sanitize(opt, getattr(group_attr, opt_name)))
|
||||
|
||||
logger.log(lvl, "*" * 80)
|
||||
|
||||
|
|
@ -983,20 +1073,31 @@ class ConfigOpts(collections.Mapping):
|
|||
if override is not None:
|
||||
return override
|
||||
|
||||
values = []
|
||||
if self._cparser is not None:
|
||||
section = group.name if group is not None else 'DEFAULT'
|
||||
try:
|
||||
return opt._get_from_config_parser(self._cparser, section)
|
||||
except (ConfigParser.NoOptionError,
|
||||
ConfigParser.NoSectionError):
|
||||
value = opt._get_from_config_parser(self._cparser, section)
|
||||
except KeyError:
|
||||
pass
|
||||
except ValueError, ve:
|
||||
except ValueError as ve:
|
||||
raise ConfigFileValueError(str(ve))
|
||||
else:
|
||||
if not opt.multi:
|
||||
# No need to continue since the last value wins
|
||||
return value[-1]
|
||||
values.extend(value)
|
||||
|
||||
name = name if group is None else group.name + '_' + name
|
||||
value = self._cli_values.get(name, None)
|
||||
value = self._cli_values.get(name)
|
||||
if value is not None:
|
||||
return value
|
||||
if not opt.multi:
|
||||
return value
|
||||
|
||||
return value + values
|
||||
|
||||
if values:
|
||||
return values
|
||||
|
||||
if default is not None:
|
||||
return default
|
||||
|
|
@ -1066,12 +1167,12 @@ class ConfigOpts(collections.Mapping):
|
|||
|
||||
:raises: ConfigFilesNotFoundError, ConfigFileParseError
|
||||
"""
|
||||
self._cparser = ConfigParser.SafeConfigParser()
|
||||
self._cparser = MultiConfigParser()
|
||||
|
||||
try:
|
||||
read_ok = self._cparser.read(config_files)
|
||||
except ConfigParser.ParsingError, cpe:
|
||||
raise ConfigFileParseError(cpe.filename, cpe.message)
|
||||
except iniparser.ParseError as pe:
|
||||
raise ConfigFileParseError(pe.filename, str(pe))
|
||||
|
||||
if read_ok != config_files:
|
||||
not_read_ok = filter(lambda f: f not in read_ok, config_files)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,126 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2012 OpenStack LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
class ParseError(Exception):
|
||||
def __init__(self, message, lineno, line):
|
||||
self.msg = message
|
||||
self.line = line
|
||||
self.lineno = lineno
|
||||
|
||||
def __str__(self):
|
||||
return 'at line %d, %s: %r' % (self.lineno, self.msg, self.line)
|
||||
|
||||
|
||||
class BaseParser(object):
|
||||
lineno = 0
|
||||
parse_exc = ParseError
|
||||
|
||||
def _assignment(self, key, value):
|
||||
self.assignment(key, value)
|
||||
return None, []
|
||||
|
||||
def _get_section(self, line):
|
||||
if line[-1] != ']':
|
||||
return self.error_no_section_end_bracket(line)
|
||||
if len(line) <= 2:
|
||||
return self.error_no_section_name(line)
|
||||
|
||||
return line[1:-1]
|
||||
|
||||
def _split_key_value(self, line):
|
||||
colon = line.find(':')
|
||||
equal = line.find('=')
|
||||
if colon < 0 and equal < 0:
|
||||
return self.error_invalid_assignment(line)
|
||||
|
||||
if colon < 0 or (equal >= 0 and equal < colon):
|
||||
key, value = line[:equal], line[equal + 1:]
|
||||
else:
|
||||
key, value = line[:colon], line[colon + 1:]
|
||||
|
||||
return key.strip(), [value.strip()]
|
||||
|
||||
def parse(self, lineiter):
|
||||
key = None
|
||||
value = []
|
||||
|
||||
for line in lineiter:
|
||||
self.lineno += 1
|
||||
|
||||
line = line.rstrip()
|
||||
if not line:
|
||||
# Blank line, ends multi-line values
|
||||
if key:
|
||||
key, value = self._assignment(key, value)
|
||||
continue
|
||||
elif line[0] in (' ', '\t'):
|
||||
# Continuation of previous assignment
|
||||
if key is None:
|
||||
self.error_unexpected_continuation(line)
|
||||
else:
|
||||
value.append(line.lstrip())
|
||||
continue
|
||||
|
||||
if key:
|
||||
# Flush previous assignment, if any
|
||||
key, value = self._assignment(key, value)
|
||||
|
||||
if line[0] == '[':
|
||||
# Section start
|
||||
section = self._get_section(line)
|
||||
if section:
|
||||
self.new_section(section)
|
||||
elif line[0] in '#;':
|
||||
self.comment(line[1:].lstrip())
|
||||
else:
|
||||
key, value = self._split_key_value(line)
|
||||
if not key:
|
||||
return self.error_empty_key(line)
|
||||
|
||||
if key:
|
||||
# Flush previous assignment, if any
|
||||
self._assignment(key, value)
|
||||
|
||||
def assignment(self, key, value):
|
||||
"""Called when a full assignment is parsed"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def new_section(self, section):
|
||||
"""Called when a new section is started"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def comment(self, comment):
|
||||
"""Called when a comment is parsed"""
|
||||
pass
|
||||
|
||||
def error_invalid_assignment(self, line):
|
||||
raise self.parse_exc("No ':' or '=' found in assignment",
|
||||
self.lineno, line)
|
||||
|
||||
def error_empty_key(self, line):
|
||||
raise self.parse_exc('Key cannot be empty', self.lineno, line)
|
||||
|
||||
def error_unexpected_continuation(self, line):
|
||||
raise self.parse_exc('Unexpected continuation line',
|
||||
self.lineno, line)
|
||||
|
||||
def error_no_section_end_bracket(self, line):
|
||||
raise self.parse_exc('Invalid section (must end with ])',
|
||||
self.lineno, line)
|
||||
|
||||
def error_no_section_name(self, line):
|
||||
raise self.parse_exc('Empty section name', self.lineno, line)
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
[DEFAULT]
|
||||
|
||||
# The list of modules to copy from openstack-common
|
||||
modules=cfg,setup
|
||||
modules=cfg,iniparser,setup
|
||||
|
||||
# The base module to hold the copy of openstack.common
|
||||
base=keystone
|
||||
|
|
|
|||
Loading…
Reference in New Issue