From ea8a0f6a8b260474151fb27c2adc9dcc88774850 Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Wed, 25 Jul 2018 20:01:37 +0100 Subject: [PATCH] Add support for looking in environment for config An _environment source is added that looks in os.environ for values. Using the environment is on by default, but can be shut down by setting `use_env` to False when __call__ is called. The enviroment is inspected before any other sources of config data but the value is used after command line arguments and before config file options. This is done by checking both the command line and config files and then inspecting the location of the result. If it is command_line, we use it. If not, we use the environment value (if any). If there's no environment value, the config file value is used. If checking the command line and config file results in a KeyError, the environment value is used, if set. The names of the environment variables follow the rules described in oslo_config.sources._environment. A new exception has been added: ConfigSourceValueError, this is the superclass of the existing ConfigFileValueError. The code in _do_get has been updated to only use ConfigFileValueError when it is in fact a file from whence a ValueError came. Documentation has been updated and a rlease note created to indicate the new functionality. Change-Id: I3245c40ebdcc96f8e3b2dc0bab3b4aa71d07ad15 --- doc/source/reference/defining.rst | 4 +- doc/source/reference/drivers.rst | 1 + doc/source/reference/locations.rst | 4 + oslo_config/cfg.py | 63 ++++++++++--- oslo_config/sources/_environment.py | 92 +++++++++++++++++++ oslo_config/tests/test_sources.py | 61 ++++++++++++ ...fig-from-environment-3feba7b4cc747d2b.yaml | 22 +++++ 7 files changed, 234 insertions(+), 13 deletions(-) create mode 100644 oslo_config/sources/_environment.py create mode 100644 releasenotes/notes/config-from-environment-3feba7b4cc747d2b.yaml diff --git a/doc/source/reference/defining.rst b/doc/source/reference/defining.rst index 65a602fe..59d80d56 100644 --- a/doc/source/reference/defining.rst +++ b/doc/source/reference/defining.rst @@ -2,7 +2,9 @@ Defining Options ================== -Configuration options may be set on the command line or in config files. +Configuration options may be set on the command line, in the +:mod:`environment `, or in config files. +Options are processed in that order. The schema for each option is defined using the :class:`Opt` class or its sub-classes, for example: diff --git a/doc/source/reference/drivers.rst b/doc/source/reference/drivers.rst index 3790675c..0c06ab17 100644 --- a/doc/source/reference/drivers.rst +++ b/doc/source/reference/drivers.rst @@ -10,3 +10,4 @@ Known Backend Drivers --------------------- .. automodule:: oslo_config.sources._uri +.. automodule:: oslo_config.sources._environment diff --git a/doc/source/reference/locations.rst b/doc/source/reference/locations.rst index bfade1ab..40b2625c 100644 --- a/doc/source/reference/locations.rst +++ b/doc/source/reference/locations.rst @@ -47,6 +47,10 @@ describing the location. Its value depends on the ``location``. - ``True`` - A value set by the user on the command line. - Empty string. + * - ``environment`` + - ``True`` + - A value set by the user in the process environment. + - The name of the environment variable. Did a user set a configuration option? ====================================== diff --git a/oslo_config/cfg.py b/oslo_config/cfg.py index a914f650..5ba83858 100644 --- a/oslo_config/cfg.py +++ b/oslo_config/cfg.py @@ -40,6 +40,8 @@ except ImportError: from oslo_config import iniparser from oslo_config import sources +# Absolute import to avoid circular import in Python 2.7 +import oslo_config.sources._environment as _environment from oslo_config import types import stevedore @@ -58,6 +60,7 @@ class Locations(enum.Enum): set_override = (3, False) user = (4, True) command_line = (5, True) + environment = (6, True) def __init__(self, num, is_user_controlled): self.num = num @@ -189,7 +192,12 @@ class ConfigFileParseError(Error): return 'Failed to parse %s: %s' % (self.config_file, self.msg) -class ConfigFileValueError(Error, ValueError): +class ConfigSourceValueError(Error, ValueError): + """Raised if a config source value does not match its opt type.""" + pass + + +class ConfigFileValueError(ConfigSourceValueError): """Raised if a config file value does not match its opt type.""" pass @@ -1946,6 +1954,9 @@ class ConfigOpts(collections.Mapping): self._validate_default_values = False self._sources = [] self._ext_mgr = None + # Though the env_driver is a Source, we load it by default. + self._use_env = True + self._env_driver = _environment.EnvironmentConfigurationSource() self.register_opt(self._config_source_opt) @@ -2009,7 +2020,7 @@ class ConfigOpts(collections.Mapping): return options def _setup(self, project, prog, version, usage, default_config_files, - default_config_dirs): + default_config_dirs, use_env): """Initialize a ConfigOpts object for option parsing.""" self._config_opts = self._make_config_options(default_config_files, default_config_dirs) @@ -2021,6 +2032,7 @@ class ConfigOpts(collections.Mapping): self.usage = usage self.default_config_files = default_config_files self.default_config_dirs = default_config_dirs + self._use_env = use_env def __clear_cache(f): @functools.wraps(f) @@ -2056,7 +2068,8 @@ class ConfigOpts(collections.Mapping): default_config_dirs=None, validate_default_values=False, description=None, - epilog=None): + epilog=None, + use_env=True): """Parse command line arguments and config files. Calling a ConfigOpts object causes the supplied command line arguments @@ -2084,6 +2097,8 @@ class ConfigOpts(collections.Mapping): :param default_config_files: config files to use by default :param default_config_dirs: config dirs to use by default :param validate_default_values: whether to validate the default values + :param use_env: If True (the default) look in the environment as one + source of option values. :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError, ConfigFilesPermissionDeniedError, RequiredOptError, DuplicateOptError @@ -2097,7 +2112,7 @@ class ConfigOpts(collections.Mapping): default_config_files, default_config_dirs) self._setup(project, prog, version, usage, default_config_files, - default_config_dirs) + default_config_dirs, use_env) self._namespace = self._parse_cli_opts(args if args is not None else sys.argv[1:]) @@ -2634,6 +2649,13 @@ class ConfigOpts(collections.Mapping): self._substitute(value, group, namespace), opt) group_name = group.name if group else None + key = (group_name, name) + + # If use_env is true, get a value from the environment but don't use + # it yet. We will look at the command line first, below. + env_val = (sources._NoValue, None) + if self._use_env: + env_val = self._env_driver.get(group_name, name, opt) if opt.mutable and namespace is None: namespace = self._mutable_ns @@ -2641,16 +2663,33 @@ class ConfigOpts(collections.Mapping): namespace = self._namespace if namespace is not None: try: - val, alt_loc = opt._get_from_namespace(namespace, group_name) - return (convert(val), alt_loc) - except KeyError: # nosec: Valid control flow instruction - pass + try: + val, alt_loc = opt._get_from_namespace(namespace, + group_name) + # Try command line first + if (val != sources._NoValue + and alt_loc.location == Locations.command_line): + return (convert(val), alt_loc) + # Environment source second + if env_val[0] != sources._NoValue: + return (convert(env_val[0]), env_val[1]) + # Default file source third + if val != sources._NoValue: + return (convert(val), alt_loc) + except KeyError: # nosec: Valid control flow instruction + # If there was a KeyError looking at config files or + # command line, retry the env_val. + if env_val[0] != sources._NoValue: + return (convert(env_val[0]), env_val[1]) except ValueError as ve: - raise ConfigFileValueError( - "Value for option %s is not valid: %s" - % (opt.name, str(ve))) + message = "Value for option %s from %s is not valid: %s" % ( + opt.name, alt_loc, str(ve)) + # Preserve backwards compatibility for file-based value + # errors. + if alt_loc.location == Locations.user: + raise ConfigFileValueError(message) + raise ConfigSourceValueError(message) - key = (group_name, name) try: return self.__drivers_cache[key] except KeyError: # nosec: Valid control flow instruction diff --git a/oslo_config/sources/_environment.py b/oslo_config/sources/_environment.py new file mode 100644 index 00000000..19dc093c --- /dev/null +++ b/oslo_config/sources/_environment.py @@ -0,0 +1,92 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +r""" +Environment +----------- + +The **environment** backend driver provides a method of accessing +configuration data in environment variables. It is enabled by default +and requires no additional configuration to use. The environment is +checked after command line options, but before configuration files. + + +Environment variables are checked for any configuration data. The variable +names take the form: + +* A prefix of ``OS_`` +* The group name, uppercased +* Separated from the option name by a `__` (double underscore) +* Followed by the name + +For an option that looks like this in the usual INI format:: + + [placement_database] + connection = sqlite:/// + +the corresponding environment variable would be +``OS_PLACEMENT_DATABASE__CONNECTION``. + +The Driver Class +================ + +.. autoclass:: EnvironmentConfigurationSourceDriver + +The Configuration Source Class +============================== + +.. autoclass:: EnvironmentConfigurationSource + +""" + +import os + +# Avoid circular import +import oslo_config.cfg +from oslo_config import sources + + +# In current practice this class is not used because the +# EnvironmentConfigurationSource is loaded by default, but we keep it +# here in case we choose to change that behavior in the future. +class EnvironmentConfigurationSourceDriver(sources.ConfigurationSourceDriver): + """A backend driver for environment variables. + + This configuration source is available by default and does not need special + configuration to use. The sample config is generated automatically but is + not necessary. + """ + + def list_options_for_discovery(self): + """There are no options for this driver.""" + return [] + + def open_source_from_opt_group(self, conf, group_name): + return EnvironmentConfigurationSource() + + +class EnvironmentConfigurationSource(sources.ConfigurationSource): + """A configuration source for options in the environment.""" + + @staticmethod + def _make_name(group_name, option_name): + group_name = group_name or 'DEFAULT' + return 'OS_{}__{}'.format(group_name.upper(), option_name.upper()) + + def get(self, group_name, option_name, opt): + env_name = self._make_name(group_name, option_name) + try: + value = os.environ[env_name] + loc = oslo_config.cfg.LocationInfo( + oslo_config.cfg.Locations.environment, env_name) + return (value, loc) + except KeyError: + return (sources._NoValue, None) diff --git a/oslo_config/tests/test_sources.py b/oslo_config/tests/test_sources.py index 8b8588fe..dca83b88 100644 --- a/oslo_config/tests/test_sources.py +++ b/oslo_config/tests/test_sources.py @@ -10,6 +10,8 @@ # License for the specific language governing permissions and limitations # under the License. +import os + from requests import HTTPError from oslo_config import _list_opts @@ -112,6 +114,65 @@ class TestLoading(base.BaseTestCase): self.assertIsNone(source) +class TestEnvironmentConfigurationSource(base.BaseTestCase): + + def setUp(self): + super(TestEnvironmentConfigurationSource, self).setUp() + self.conf = cfg.ConfigOpts() + self.conf_fixture = self.useFixture(fixture.Config(self.conf)) + self.conf.register_opt(cfg.StrOpt('bar'), 'foo') + + def cleanup(): + if 'OS_FOO__BAR' in os.environ: + del os.environ['OS_FOO__BAR'] + self.addCleanup(cleanup) + + def test_simple_environment_get(self): + self.conf(args=[]) + env_value = 'goodbye' + os.environ['OS_FOO__BAR'] = env_value + + self.assertEqual(env_value, self.conf['foo']['bar']) + + def test_env_beats_files(self): + file_value = 'hello' + env_value = 'goodbye' + self.conf(args=[]) + self.conf_fixture.load_raw_values( + group='foo', + bar=file_value, + ) + + self.assertEqual(file_value, self.conf['foo']['bar']) + self.conf.reload_config_files() + os.environ['OS_FOO__BAR'] = env_value + self.assertEqual(env_value, self.conf['foo']['bar']) + + def test_cli_beats_env(self): + env_value = 'goodbye' + cli_value = 'cli' + os.environ['OS_FOO__BAR'] = env_value + self.conf.register_cli_opt(cfg.StrOpt('bar'), 'foo') + self.conf(args=['--foo=%s' % cli_value]) + + self.assertEqual(cli_value, self.conf['foo']['bar']) + + def test_use_env_false_allows_files(self): + file_value = 'hello' + env_value = 'goodbye' + os.environ['OS_FOO__BAR'] = env_value + self.conf(args=[], use_env=False) + self.conf_fixture.load_raw_values( + group='foo', + bar=file_value, + ) + + self.assertEqual(file_value, self.conf['foo']['bar']) + self.conf.reset() + self.conf(args=[], use_env=True) + self.assertEqual(env_value, self.conf['foo']['bar']) + + def make_uri(name): return "https://oslo.config/{}.conf".format(name) diff --git a/releasenotes/notes/config-from-environment-3feba7b4cc747d2b.yaml b/releasenotes/notes/config-from-environment-3feba7b4cc747d2b.yaml new file mode 100644 index 00000000..d78337e6 --- /dev/null +++ b/releasenotes/notes/config-from-environment-3feba7b4cc747d2b.yaml @@ -0,0 +1,22 @@ +--- +features: + - | + Support for accessing configuration data in environment variables via the + environment backend driver, enabled by default. The environment is checked + after command line options, but before configuration files. + + Environment variables are checked for any configuration data. The variable + names take the form: + + * A prefix of ``OS_`` + * The group name, uppercased + * Separated from the option name by a `__` (double underscore) + * Followed by the name + + For an option that looks like this in the usual INI format:: + + [placement_database] + connection = sqlite:/// + + the corresponding environment variable would be + ``OS_PLACEMENT_DATABASE__CONNECTION``.