From c49aacec276b990891ecb4a570c3977dda313792 Mon Sep 17 00:00:00 2001 From: Kristi Nikolla Date: Tue, 30 May 2017 20:45:41 -0400 Subject: [PATCH] Config refactoring This splits the configuration module into a package with multiple modules. Additionally adds support for oslo-config-generator. Additionally adds mixmatch.conf to .gitignore Change-Id: I735c6e852ef3cddb6dff654e1b8777ba239b91ab --- .gitignore | 4 +- etc/mixmatch.conf.sample | 29 +---- mixmatch/auth.py | 20 ++-- mixmatch/config.py | 169 --------------------------- mixmatch/config/__init__.py | 89 ++++++++++++++ mixmatch/config/auth.py | 58 +++++++++ mixmatch/config/cache.py | 49 ++++++++ mixmatch/config/default.py | 51 ++++++++ mixmatch/config/service_providers.py | 64 ++++++++++ mixmatch/listener.py | 7 +- mixmatch/proxy.py | 7 +- mixmatch/services.py | 2 +- mixmatch/tests/unit/base.py | 6 +- setup.cfg | 4 + 14 files changed, 343 insertions(+), 216 deletions(-) delete mode 100644 mixmatch/config.py create mode 100644 mixmatch/config/__init__.py create mode 100644 mixmatch/config/auth.py create mode 100644 mixmatch/config/cache.py create mode 100644 mixmatch/config/default.py create mode 100644 mixmatch/config/service_providers.py diff --git a/.gitignore b/.gitignore index 963e589..b3898b5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +mixmatch.conf + *.py[cod] # C extensions @@ -55,4 +57,4 @@ ChangeLog .*sw? # Files created by releasenotes build -releasenotes/build \ No newline at end of file +releasenotes/build diff --git a/etc/mixmatch.conf.sample b/etc/mixmatch.conf.sample index c05ab3a..e31c488 100644 --- a/etc/mixmatch.conf.sample +++ b/etc/mixmatch.conf.sample @@ -20,6 +20,8 @@ image_api_versions = v2.3, v2.2, v2.1, v2.0, v1.1, v1.0 # Available volume API versions volume_api_versions = v3.0, v2.0, v1.0 +debug=False + #------------------------------------------------------------------------------ [database] @@ -105,30 +107,3 @@ volume_endpoint="http://192.168.0.141:8776" # Services that are enabled for this service provider enabled_services=image, volume - -#------------------------------------------------------------------------------ - -# Logging -[loggers] - -keys = root - -#------------------------------------------------------------------------------ - -[handlers] - -keys = stdout - -#------------------------------------------------------------------------------ - -[formatters] - -keys = default - -#------------------------------------------------------------------------------ - -[logger_root] - -level = DEBUG -handlers = stdout -formatter = default diff --git a/mixmatch/auth.py b/mixmatch/auth.py index e803c6f..4ec8284 100644 --- a/mixmatch/auth.py +++ b/mixmatch/auth.py @@ -22,10 +22,14 @@ import json from flask import abort from mixmatch import config -from mixmatch.config import LOG, CONF, get_conf_for_sp + +CONF = config.CONF +LOG = config.LOG + +MEMOIZE_SESSION = config.auth.MEMOIZE -@config.MEMOIZE_SESSION +@MEMOIZE_SESSION def get_client(): """Return a Keystone client capable of validating tokens.""" LOG.info("Getting Admin Client") @@ -41,7 +45,7 @@ def get_client(): return v3.client.Client(session=local_session) -@config.MEMOIZE_SESSION +@MEMOIZE_SESSION def get_local_auth(user_token): """Return a Keystone session for the local cluster.""" LOG.debug("Getting session for %s" % user_token) @@ -62,10 +66,10 @@ def get_local_auth(user_token): return session.Session(auth=local_auth) -@config.MEMOIZE_SESSION +@MEMOIZE_SESSION def get_unscoped_sp_auth(service_provider, user_token): """Perform K2K auth, and return an unscoped session.""" - conf = get_conf_for_sp(service_provider) + conf = config.service_providers.get(CONF, service_provider) local_auth = get_local_auth(user_token).auth LOG.debug("Getting unscoped session for (%s, %s)" % (service_provider, user_token)) @@ -78,17 +82,17 @@ def get_unscoped_sp_auth(service_provider, user_token): def get_projects_at_sp(service_provider, user_token): """Perform K2K auth, and return the projects that can be scoped to.""" - conf = get_conf_for_sp(service_provider) + conf = config.service_providers.get(CONF, service_provider) unscoped_session = get_unscoped_sp_auth(service_provider, user_token) r = json.loads(str(unscoped_session.get( conf.auth_url + "/OS-FEDERATION/projects").text)) return [project[u'id'] for project in r[u'projects']] -@config.MEMOIZE_SESSION +@MEMOIZE_SESSION def get_sp_auth(service_provider, user_token, remote_project_id): """Perform K2K auth, and return a session for a remote cluster.""" - conf = get_conf_for_sp(service_provider) + conf = config.service_providers.get(CONF, service_provider) local_auth = get_local_auth(user_token).auth LOG.debug("Getting session for (%s, %s, %s)" % (service_provider, diff --git a/mixmatch/config.py b/mixmatch/config.py deleted file mode 100644 index 87c5ba4..0000000 --- a/mixmatch/config.py +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright 2016 Massachusetts Open Cloud -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from os import path - -from oslo_config import cfg -from oslo_log import log -from oslo_cache import core as cache - -LOG = log.getLogger('root') - -CONF = cfg.CONF - - -# Proxy - -proxy_opts = [ - cfg.IntOpt('port', - default=5001, - help='Web Server Port'), - - cfg.ListOpt('service_providers', - default=[], - help='List of service providers'), - - cfg.BoolOpt('search_by_broadcast', - default=False, - help='Search All Service Providers on Unknown Resource ID'), - - cfg.BoolOpt('aggregation', - default=False, - help='Enable Aggregation when listing resources.'), - - cfg.BoolOpt('caching', - default=True, - help='Enable token caching using oslo.cache'), - - cfg.IntOpt('cache_time', - default=600, - help='How long to store cached tokens for'), - - cfg.ListOpt('image_api_versions', - default=['v2.3', 'v2.2', 'v2.1', 'v2.0', 'v1.1', 'v1.0'], - help='List of supported image api versions'), - - cfg.ListOpt('volume_api_versions', - default=['v3.0', 'v2.0', 'v1.0'], - help='List of supported volume api versions'), -] - -# Keystone -auth_group = cfg.OptGroup(name='auth', title='Keystone Config Group') - -auth_opts = [ - cfg.StrOpt('auth_url', - default='http://localhost:35357/v3', - help='Keystone AUTH URL'), - - cfg.StrOpt('username', - default='admin', - help='Proxy username'), - - cfg.StrOpt('user_domain_id', - default='default', - help='Proxy user domain id'), - - cfg.StrOpt('password', - default='nomoresecrete', - help='Proxy user password'), - - cfg.StrOpt('project_name', - default='admin', - help='Proxy project name'), - - cfg.StrOpt('project_domain_id', - default='default', - help='Proxy project domain id') -] - - -CONF.register_opts(proxy_opts) - -CONF.register_group(auth_group) -CONF.register_opts(auth_opts, auth_group) - -# Logging -log.register_options(CONF) - -# Caching -cache.configure(CONF) - -MEMOIZE_SESSION = None -session_cache_region = cache.create_region() - -MEMOIZE_SESSION = cache.get_memoization_decorator( - CONF, session_cache_region, group='auth') - - -def load_config(): - """Load parameters from the proxy's config file.""" - conf_files = [f for f in ['mixmatch.conf', - 'etc/mixmatch/mixmatch.conf', - '/etc/mixmatch/mixmatch.conf'] if path.isfile(f)] - if conf_files is not []: - CONF(default_config_files=conf_files) - - -def more_config(): - """Perform configuration that must be delayed until after import time. - - This code must be delayed until the config files have been loaded. They - are in a separate file so that unit tests can run them without loading - configuration from a file. - """ - cache.configure_cache_region(CONF, session_cache_region) - - for service_provider in CONF.service_providers: - - sp_group = cfg.OptGroup(name='sp_%s' % service_provider, - title=service_provider) - sp_opts = [ - cfg.StrOpt('sp_name', - default="default", - help='SP ID in Keystone Catalog. Omit for local.'), - - cfg.StrOpt('messagebus', - help='URI to connect to message bus'), - - cfg.StrOpt('services', - default=None, - help='Enabled services for this service provider.'), - - cfg.StrOpt('auth_url', - default=None, - help='Keystone AUTH URL for Service Provider'), - - cfg.StrOpt('image_endpoint', - default=None, - help="Image Endpoint for Service Provider"), - - cfg.StrOpt('volume_endpoint', - default=None, - help="Volume Endpoint for Service Provider"), - - cfg.ListOpt('enabled_services', - default=['image', 'volume'], - help="Services to enable for Service Provider") - ] - - CONF.register_group(sp_group) - CONF.register_opts(sp_opts, sp_group) - - log.setup(CONF, 'demo') - - -def get_conf_for_sp(sp_id): - """Get the configuration opject for a specifc service provider.""" - return CONF.__getattr__('sp_%s' % sp_id) diff --git a/mixmatch/config/__init__.py b/mixmatch/config/__init__.py new file mode 100644 index 0000000..7ee9025 --- /dev/null +++ b/mixmatch/config/__init__.py @@ -0,0 +1,89 @@ +# Copyright 2017 Massachusetts Open Cloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from os import path + +from oslo_config import cfg +from oslo_log import log + +from mixmatch.config import auth +from mixmatch.config import cache +from mixmatch.config import default +from mixmatch.config import service_providers + +LOG = log.getLogger('root') +CONF = cfg.CONF + +# Note(knikolla): Configuration modules are registered in the list below. +# Order matters, and they are loaded in the defined order. +# +# Each module must define the following variables and functions. +# * GROUP - oslo_config.cfg.OptGroup or None +# * OPTS - list of oslo_config.cfg.{Int,Str,Bool,etc}Opt or [] +# * pre_config() - function is executed at module import time before OPTS +# are registered. +# * post_config() - function is executed after the configuration files have +# been loaded. +MODULES = [ + default, + cache, + auth, + service_providers +] + + +def load_from_file(): + """Load options from the configuration file.""" + conf_files = [f for f in ['mixmatch.conf', + 'etc/mixmatch/mixmatch.conf', + '/etc/mixmatch/mixmatch.conf'] if path.isfile(f)] + if conf_files is not []: + CONF(default_config_files=conf_files) + + +def register_opts(): + for option_module in MODULES: + if option_module.GROUP: + CONF.register_group(option_module.GROUP) + + if option_module.OPTS: + CONF.register_opts(option_module.OPTS, option_module.GROUP) + + +def list_opts(): + return [(m.GROUP, m.OPTS) for m in MODULES] + + +def pre_config(): + log.register_options(CONF) + + for option_module in MODULES: + option_module.pre_config(CONF) + + register_opts() + + +def post_config(): + for option_module in MODULES: + option_module.post_config(CONF) + + log.setup(CONF, 'demo') + + +def configure(): + load_from_file() + post_config() + + +pre_config() diff --git a/mixmatch/config/auth.py b/mixmatch/config/auth.py new file mode 100644 index 0000000..5ac026e --- /dev/null +++ b/mixmatch/config/auth.py @@ -0,0 +1,58 @@ +# Copyright 2017 Massachusetts Open Cloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from mixmatch.config import cache + +GROUP = cfg.OptGroup(name='auth', title='Keystone Config Group') + +OPTS = [ + cfg.StrOpt('auth_url', + default='http://localhost:35357/v3', + help='Keystone AUTH URL'), + + cfg.StrOpt('username', + default='admin', + help='Proxy username'), + + cfg.StrOpt('user_domain_id', + default='default', + help='Proxy user domain id'), + + cfg.StrOpt('password', + default='nomoresecrete', + help='Proxy user password'), + + cfg.StrOpt('project_name', + default='admin', + help='Proxy project name'), + + cfg.StrOpt('project_domain_id', + default='default', + help='Proxy project domain id') +] + +MEMOIZE = None + + +def pre_config(conf): + global MEMOIZE + + cache.register_region('auth') + MEMOIZE = cache.get_decorator(conf, name='auth', group='auth') + + +def post_config(conf): + pass diff --git a/mixmatch/config/cache.py b/mixmatch/config/cache.py new file mode 100644 index 0000000..495b3ae --- /dev/null +++ b/mixmatch/config/cache.py @@ -0,0 +1,49 @@ +# Copyright 2017 Massachusetts Open Cloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_cache import core + +GROUP = None + +OPTS = [ + cfg.BoolOpt('caching', + default=True, + help='Enable caching.'), + + cfg.IntOpt('cache_time', + default=600, + help='How long to cache things.'), +] + +_REGIONS = {} + + +def pre_config(conf): + core.configure(conf) + + +def post_config(conf): + for _, region in _REGIONS.items(): + core.configure_cache_region(conf, region) + + +def register_region(name): + _REGIONS[name] = core.create_region() + + +def get_decorator(conf, name, group): + return core.get_memoization_decorator(conf, + _REGIONS[name], + group=group) diff --git a/mixmatch/config/default.py b/mixmatch/config/default.py new file mode 100644 index 0000000..d439025 --- /dev/null +++ b/mixmatch/config/default.py @@ -0,0 +1,51 @@ +# Copyright 2017 Massachusetts Open Cloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +GROUP = None + +OPTS = [ + cfg.IntOpt('port', + default=5001, + help='Web Server Port'), + + cfg.ListOpt('service_providers', + default=[], + help='List of service providers'), + + cfg.BoolOpt('search_by_broadcast', + default=False, + help='Search All Service Providers on Unknown Resource ID'), + + cfg.BoolOpt('aggregation', + default=False, + help='Enable Aggregation when listing resources.'), + + cfg.ListOpt('image_api_versions', + default=['v2.3', 'v2.2', 'v2.1', 'v2.0', 'v1.1', 'v1.0'], + help='List of supported image api versions'), + + cfg.ListOpt('volume_api_versions', + default=['v3.0', 'v2.0', 'v1.0'], + help='List of supported volume api versions'), +] + + +def pre_config(conf): + pass + + +def post_config(conf): + pass diff --git a/mixmatch/config/service_providers.py b/mixmatch/config/service_providers.py new file mode 100644 index 0000000..db6401f --- /dev/null +++ b/mixmatch/config/service_providers.py @@ -0,0 +1,64 @@ +# Copyright 2017 Massachusetts Open Cloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +GROUP = None + +OPTS = [] + +SP_OPTS = [ + cfg.StrOpt('sp_name', + default="default", + help='SP ID in Keystone Catalog. Omit for local.'), + + cfg.StrOpt('messagebus', + help='URI to connect to message bus'), + + cfg.StrOpt('services', + default=None, + help='Enabled services for this service provider.'), + + cfg.StrOpt('auth_url', + default=None, + help='Keystone AUTH URL for Service Provider'), + + cfg.StrOpt('image_endpoint', + default=None, + help="Image Endpoint for Service Provider"), + + cfg.StrOpt('volume_endpoint', + default=None, + help="Volume Endpoint for Service Provider"), + + cfg.ListOpt('enabled_services', + default=['image', 'volume'], + help="Services to enable for Service Provider") +] + + +def pre_config(conf): + pass + + +def post_config(conf): + for service_provider in conf.service_providers: + sp_group = cfg.OptGroup(name='sp_%s' % service_provider, + title=service_provider) + conf.register_opts(SP_OPTS, sp_group) + + +def get(conf, sp_id): + """Get the configuration opject for a specifc service provider.""" + return conf.__getattr__('sp_%s' % sp_id) diff --git a/mixmatch/listener.py b/mixmatch/listener.py index d7b96a0..46c99fc 100644 --- a/mixmatch/listener.py +++ b/mixmatch/listener.py @@ -15,7 +15,7 @@ import oslo_messaging from mixmatch import config -from mixmatch.config import CONF, LOG +from mixmatch.config import CONF, LOG, service_providers from mixmatch import model from mixmatch.model import insert, delete, ResourceMapping @@ -170,7 +170,7 @@ def get_server_for_sp(sp): The server can be run in the background under eventlet using .start() """ - cfg = config.get_conf_for_sp(sp) + cfg = service_providers.get(CONF, sp) transport = oslo_messaging.get_notification_transport(CONF, cfg.messagebus) targets = [oslo_messaging.Target(topic='notifications')] return oslo_messaging.get_notification_listener( @@ -182,8 +182,7 @@ def get_server_for_sp(sp): if __name__ == "__main__": - config.load_config() - config.more_config() + config.configure() model.create_tables() LOG.info("Now listening for changes") diff --git a/mixmatch/proxy.py b/mixmatch/proxy.py index affc970..ed790db 100644 --- a/mixmatch/proxy.py +++ b/mixmatch/proxy.py @@ -18,7 +18,7 @@ import flask from flask import abort from mixmatch import config -from mixmatch.config import LOG, CONF +from mixmatch.config import LOG, CONF, service_providers from mixmatch.session import app from mixmatch.session import chunked_reader from mixmatch.session import request @@ -97,7 +97,7 @@ class RequestHandler(object): self.enabled_sps = filter( lambda sp: (self.details['service'] in - config.get_conf_for_sp(sp).enabled_services), + service_providers.get(CONF, sp).enabled_services), CONF.service_providers ) @@ -332,8 +332,7 @@ def proxy(path): def main(): - config.load_config() - config.more_config() + config.configure() model.create_tables() diff --git a/mixmatch/services.py b/mixmatch/services.py index 50ef692..532a423 100644 --- a/mixmatch/services.py +++ b/mixmatch/services.py @@ -25,7 +25,7 @@ CONF = config.CONF def construct_url(service_provider, service_type, version=None, action=None, project_id=None): """Construct the full URL for an Openstack API call.""" - conf = config.get_conf_for_sp(service_provider) + conf = config.service_providers.get(CONF, service_provider) if service_type == 'image': url = conf.image_endpoint diff --git a/mixmatch/tests/unit/base.py b/mixmatch/tests/unit/base.py index cc4974d..4ad7ce9 100644 --- a/mixmatch/tests/unit/base.py +++ b/mixmatch/tests/unit/base.py @@ -21,10 +21,12 @@ from six.moves.urllib import parse from requests_mock.contrib import fixture as requests_fixture from oslo_config import fixture as config_fixture -from mixmatch.config import CONF, more_config +from mixmatch import config from mixmatch.proxy import app from mixmatch.model import BASE, enginefacade +CONF = config.CONF + class BaseTest(testcase.TestCase): def setUp(self): @@ -58,7 +60,7 @@ class BaseTest(testcase.TestCase): group='sp_remote1', image_endpoint='http://images.remote1', volume_endpoint='http://volumes.remote1') - more_config() + config.post_config() def load_auth_fixtures(self): self.auth = FakeSession(token=uuid.uuid4().hex, diff --git a/setup.cfg b/setup.cfg index 3310d6b..0ed80d7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,10 @@ packages = data_files = etc/ = etc/* +[entry_points] +oslo.config.opts = + mixmatch = mixmatch.config:list_opts + [build_sphinx] source-dir = doc/source build-dir = doc/build