x
/
mixmatch
Code Issues Proposed changes

Config refactoring 

This splits the configuration module into a package with multiple
modules.

Additionally adds support for oslo-config-generator.
Additionally adds mixmatch.conf to .gitignore

Change-Id: I735c6e852ef3cddb6dff654e1b8777ba239b91ab
This commit is contained in:
Kristi Nikolla 2017-05-30 20:45:41 -04:00
parent 32e70f0d7f
commit c49aacec27
14 changed files with 343 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -1,3 +1,5 @@
mixmatch.conf
*.py[cod]
# C extensions
@ -55,4 +57,4 @@ ChangeLog
.*sw?
# Files created by releasenotes build
releasenotes/build
releasenotes/build

29
etc/mixmatch.conf.sample
View File

@ -20,6 +20,8 @@ image_api_versions = v2.3, v2.2, v2.1, v2.0, v1.1, v1.0
# Available volume API versions
volume_api_versions = v3.0, v2.0, v1.0
debug=False
#------------------------------------------------------------------------------
[database]
@ -105,30 +107,3 @@ volume_endpoint="http://192.168.0.141:8776"
# Services that are enabled for this service provider
enabled_services=image, volume
#------------------------------------------------------------------------------
# Logging
[loggers]
keys = root
#------------------------------------------------------------------------------
[handlers]
keys = stdout
#------------------------------------------------------------------------------
[formatters]
keys = default
#------------------------------------------------------------------------------
[logger_root]
level = DEBUG
handlers = stdout
formatter = default

20
mixmatch/auth.py
View File

@ -22,10 +22,14 @@ import json
from flask import abort
from mixmatch import config
from mixmatch.config import LOG, CONF, get_conf_for_sp
CONF = config.CONF
LOG = config.LOG
MEMOIZE_SESSION = config.auth.MEMOIZE
@config.MEMOIZE_SESSION
@MEMOIZE_SESSION
def get_client():
"""Return a Keystone client capable of validating tokens."""
LOG.info("Getting Admin Client")
@ -41,7 +45,7 @@ def get_client():
return v3.client.Client(session=local_session)
@config.MEMOIZE_SESSION
@MEMOIZE_SESSION
def get_local_auth(user_token):
"""Return a Keystone session for the local cluster."""
LOG.debug("Getting session for %s" % user_token)
@ -62,10 +66,10 @@ def get_local_auth(user_token):
return session.Session(auth=local_auth)
@config.MEMOIZE_SESSION
@MEMOIZE_SESSION
def get_unscoped_sp_auth(service_provider, user_token):
"""Perform K2K auth, and return an unscoped session."""
conf = get_conf_for_sp(service_provider)
conf = config.service_providers.get(CONF, service_provider)
local_auth = get_local_auth(user_token).auth
LOG.debug("Getting unscoped session for (%s, %s)" % (service_provider,
user_token))
@ -78,17 +82,17 @@ def get_unscoped_sp_auth(service_provider, user_token):
def get_projects_at_sp(service_provider, user_token):
"""Perform K2K auth, and return the projects that can be scoped to."""
conf = get_conf_for_sp(service_provider)
conf = config.service_providers.get(CONF, service_provider)
unscoped_session = get_unscoped_sp_auth(service_provider, user_token)
r = json.loads(str(unscoped_session.get(
conf.auth_url + "/OS-FEDERATION/projects").text))
return [project[u'id'] for project in r[u'projects']]
@config.MEMOIZE_SESSION
@MEMOIZE_SESSION
def get_sp_auth(service_provider, user_token, remote_project_id):
"""Perform K2K auth, and return a session for a remote cluster."""
conf = get_conf_for_sp(service_provider)
conf = config.service_providers.get(CONF, service_provider)
local_auth = get_local_auth(user_token).auth
LOG.debug("Getting session for (%s, %s, %s)" % (service_provider,

169
mixmatch/config.py
View File

@ -1,169 +0,0 @@
# Copyright 2016 Massachusetts Open Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from os import path
from oslo_config import cfg
from oslo_log import log
from oslo_cache import core as cache
LOG = log.getLogger('root')
CONF = cfg.CONF
# Proxy
proxy_opts = [
cfg.IntOpt('port',
default=5001,
help='Web Server Port'),
cfg.ListOpt('service_providers',
default=[],
help='List of service providers'),
cfg.BoolOpt('search_by_broadcast',
default=False,
help='Search All Service Providers on Unknown Resource ID'),
cfg.BoolOpt('aggregation',
default=False,
help='Enable Aggregation when listing resources.'),
cfg.BoolOpt('caching',
default=True,
help='Enable token caching using oslo.cache'),
cfg.IntOpt('cache_time',
default=600,
help='How long to store cached tokens for'),
cfg.ListOpt('image_api_versions',
default=['v2.3', 'v2.2', 'v2.1', 'v2.0', 'v1.1', 'v1.0'],
help='List of supported image api versions'),
cfg.ListOpt('volume_api_versions',
default=['v3.0', 'v2.0', 'v1.0'],
help='List of supported volume api versions'),
]
# Keystone
auth_group = cfg.OptGroup(name='auth', title='Keystone Config Group')
auth_opts = [
cfg.StrOpt('auth_url',
default='http://localhost:35357/v3',
help='Keystone AUTH URL'),
cfg.StrOpt('username',
default='admin',
help='Proxy username'),
cfg.StrOpt('user_domain_id',
default='default',
help='Proxy user domain id'),
cfg.StrOpt('password',
default='nomoresecrete',
help='Proxy user password'),
cfg.StrOpt('project_name',
default='admin',
help='Proxy project name'),
cfg.StrOpt('project_domain_id',
default='default',
help='Proxy project domain id')
]
CONF.register_opts(proxy_opts)
CONF.register_group(auth_group)
CONF.register_opts(auth_opts, auth_group)
# Logging
log.register_options(CONF)
# Caching
cache.configure(CONF)
MEMOIZE_SESSION = None
session_cache_region = cache.create_region()
MEMOIZE_SESSION = cache.get_memoization_decorator(
CONF, session_cache_region, group='auth')
def load_config():
"""Load parameters from the proxy's config file."""
conf_files = [f for f in ['mixmatch.conf',
'etc/mixmatch/mixmatch.conf',
'/etc/mixmatch/mixmatch.conf'] if path.isfile(f)]
if conf_files is not []:
CONF(default_config_files=conf_files)
def more_config():
"""Perform configuration that must be delayed until after import time.
This code must be delayed until the config files have been loaded. They
are in a separate file so that unit tests can run them without loading
configuration from a file.
"""
cache.configure_cache_region(CONF, session_cache_region)
for service_provider in CONF.service_providers:
sp_group = cfg.OptGroup(name='sp_%s' % service_provider,
title=service_provider)
sp_opts = [
cfg.StrOpt('sp_name',
default="default",
help='SP ID in Keystone Catalog. Omit for local.'),
cfg.StrOpt('messagebus',
help='URI to connect to message bus'),
cfg.StrOpt('services',
default=None,
help='Enabled services for this service provider.'),
cfg.StrOpt('auth_url',
default=None,
help='Keystone AUTH URL for Service Provider'),
cfg.StrOpt('image_endpoint',
default=None,
help="Image Endpoint for Service Provider"),
cfg.StrOpt('volume_endpoint',
default=None,
help="Volume Endpoint for Service Provider"),
cfg.ListOpt('enabled_services',
default=['image', 'volume'],
help="Services to enable for Service Provider")
]
CONF.register_group(sp_group)
CONF.register_opts(sp_opts, sp_group)
log.setup(CONF, 'demo')
def get_conf_for_sp(sp_id):
"""Get the configuration opject for a specifc service provider."""
return CONF.__getattr__('sp_%s' % sp_id)

89
mixmatch/config/__init__.py Normal file
View File

@ -0,0 +1,89 @@
# Copyright 2017 Massachusetts Open Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from os import path
from oslo_config import cfg
from oslo_log import log
from mixmatch.config import auth
from mixmatch.config import cache
from mixmatch.config import default
from mixmatch.config import service_providers
LOG = log.getLogger('root')
CONF = cfg.CONF
# Note(knikolla): Configuration modules are registered in the list below.
# Order matters, and they are loaded in the defined order.
#
# Each module must define the following variables and functions.
# * GROUP - oslo_config.cfg.OptGroup or None
# * OPTS - list of oslo_config.cfg.{Int,Str,Bool,etc}Opt or []
# * pre_config() - function is executed at module import time before OPTS
# are registered.
# * post_config() - function is executed after the configuration files have
# been loaded.
MODULES = [
default,
cache,
auth,
service_providers
]
def load_from_file():
"""Load options from the configuration file."""
conf_files = [f for f in ['mixmatch.conf',
'etc/mixmatch/mixmatch.conf',
'/etc/mixmatch/mixmatch.conf'] if path.isfile(f)]
if conf_files is not []:
CONF(default_config_files=conf_files)
def register_opts():
for option_module in MODULES:
if option_module.GROUP:
CONF.register_group(option_module.GROUP)
if option_module.OPTS:
CONF.register_opts(option_module.OPTS, option_module.GROUP)
def list_opts():
return [(m.GROUP, m.OPTS) for m in MODULES]
def pre_config():
log.register_options(CONF)
for option_module in MODULES:
option_module.pre_config(CONF)
register_opts()
def post_config():
for option_module in MODULES:
option_module.post_config(CONF)
log.setup(CONF, 'demo')
def configure():
load_from_file()
post_config()
pre_config()

58
mixmatch/config/auth.py Normal file
View File

@ -0,0 +1,58 @@
# Copyright 2017 Massachusetts Open Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from mixmatch.config import cache
GROUP = cfg.OptGroup(name='auth', title='Keystone Config Group')
OPTS = [
cfg.StrOpt('auth_url',
default='http://localhost:35357/v3',
help='Keystone AUTH URL'),
cfg.StrOpt('username',
default='admin',
help='Proxy username'),
cfg.StrOpt('user_domain_id',
default='default',
help='Proxy user domain id'),
cfg.StrOpt('password',
default='nomoresecrete',
help='Proxy user password'),
cfg.StrOpt('project_name',
default='admin',
help='Proxy project name'),
cfg.StrOpt('project_domain_id',
default='default',
help='Proxy project domain id')
]
MEMOIZE = None
def pre_config(conf):
global MEMOIZE
cache.register_region('auth')
MEMOIZE = cache.get_decorator(conf, name='auth', group='auth')
def post_config(conf):
pass

49
mixmatch/config/cache.py Normal file
View File

@ -0,0 +1,49 @@
# Copyright 2017 Massachusetts Open Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from oslo_cache import core
GROUP = None
OPTS = [
cfg.BoolOpt('caching',
default=True,
help='Enable caching.'),
cfg.IntOpt('cache_time',
default=600,
help='How long to cache things.'),
]
_REGIONS = {}
def pre_config(conf):
core.configure(conf)
def post_config(conf):
for _, region in _REGIONS.items():
core.configure_cache_region(conf, region)
def register_region(name):
_REGIONS[name] = core.create_region()
def get_decorator(conf, name, group):
return core.get_memoization_decorator(conf,
_REGIONS[name],
group=group)

51
mixmatch/config/default.py Normal file
View File

@ -0,0 +1,51 @@
# Copyright 2017 Massachusetts Open Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
GROUP = None
OPTS = [
cfg.IntOpt('port',
default=5001,
help='Web Server Port'),
cfg.ListOpt('service_providers',
default=[],
help='List of service providers'),
cfg.BoolOpt('search_by_broadcast',
default=False,
help='Search All Service Providers on Unknown Resource ID'),
cfg.BoolOpt('aggregation',
default=False,
help='Enable Aggregation when listing resources.'),
cfg.ListOpt('image_api_versions',
default=['v2.3', 'v2.2', 'v2.1', 'v2.0', 'v1.1', 'v1.0'],
help='List of supported image api versions'),
cfg.ListOpt('volume_api_versions',
default=['v3.0', 'v2.0', 'v1.0'],
help='List of supported volume api versions'),
]
def pre_config(conf):
pass
def post_config(conf):
pass

64
mixmatch/config/service_providers.py Normal file
View File

@ -0,0 +1,64 @@
# Copyright 2017 Massachusetts Open Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
GROUP = None
OPTS = []
SP_OPTS = [
cfg.StrOpt('sp_name',
default="default",
help='SP ID in Keystone Catalog. Omit for local.'),
cfg.StrOpt('messagebus',
help='URI to connect to message bus'),
cfg.StrOpt('services',
default=None,
help='Enabled services for this service provider.'),
cfg.StrOpt('auth_url',
default=None,
help='Keystone AUTH URL for Service Provider'),
cfg.StrOpt('image_endpoint',
default=None,
help="Image Endpoint for Service Provider"),
cfg.StrOpt('volume_endpoint',
default=None,
help="Volume Endpoint for Service Provider"),
cfg.ListOpt('enabled_services',
default=['image', 'volume'],
help="Services to enable for Service Provider")
]
def pre_config(conf):
pass
def post_config(conf):
for service_provider in conf.service_providers:
sp_group = cfg.OptGroup(name='sp_%s' % service_provider,
title=service_provider)
conf.register_opts(SP_OPTS, sp_group)
def get(conf, sp_id):
"""Get the configuration opject for a specifc service provider."""
return conf.__getattr__('sp_%s' % sp_id)

7
mixmatch/listener.py
View File

@ -15,7 +15,7 @@
import oslo_messaging
from mixmatch import config
from mixmatch.config import CONF, LOG
from mixmatch.config import CONF, LOG, service_providers
from mixmatch import model
from mixmatch.model import insert, delete, ResourceMapping
@ -170,7 +170,7 @@ def get_server_for_sp(sp):
The server can be run in the background under eventlet using .start()
"""
cfg = config.get_conf_for_sp(sp)
cfg = service_providers.get(CONF, sp)
transport = oslo_messaging.get_notification_transport(CONF, cfg.messagebus)
targets = [oslo_messaging.Target(topic='notifications')]
return oslo_messaging.get_notification_listener(
@ -182,8 +182,7 @@ def get_server_for_sp(sp):
if __name__ == "__main__":
config.load_config()
config.more_config()
config.configure()
model.create_tables()
LOG.info("Now listening for changes")

7
mixmatch/proxy.py
View File

@ -18,7 +18,7 @@ import flask
from flask import abort
from mixmatch import config
from mixmatch.config import LOG, CONF
from mixmatch.config import LOG, CONF, service_providers
from mixmatch.session import app
from mixmatch.session import chunked_reader
from mixmatch.session import request
@ -97,7 +97,7 @@ class RequestHandler(object):
self.enabled_sps = filter(
lambda sp: (self.details['service'] in
config.get_conf_for_sp(sp).enabled_services),
service_providers.get(CONF, sp).enabled_services),
CONF.service_providers
)
@ -332,8 +332,7 @@ def proxy(path):
def main():
config.load_config()
config.more_config()
config.configure()
model.create_tables()

2
mixmatch/services.py
View File

@ -25,7 +25,7 @@ CONF = config.CONF
def construct_url(service_provider, service_type,
version=None, action=None, project_id=None):
"""Construct the full URL for an Openstack API call."""
conf = config.get_conf_for_sp(service_provider)
conf = config.service_providers.get(CONF, service_provider)
if service_type == 'image':
url = conf.image_endpoint

6
mixmatch/tests/unit/base.py
View File

@ -21,10 +21,12 @@ from six.moves.urllib import parse
from requests_mock.contrib import fixture as requests_fixture
from oslo_config import fixture as config_fixture
from mixmatch.config import CONF, more_config
from mixmatch import config
from mixmatch.proxy import app
from mixmatch.model import BASE, enginefacade
CONF = config.CONF
class BaseTest(testcase.TestCase):
def setUp(self):
@ -58,7 +60,7 @@ class BaseTest(testcase.TestCase):
group='sp_remote1',
image_endpoint='http://images.remote1',
volume_endpoint='http://volumes.remote1')
more_config()
config.post_config()
def load_auth_fixtures(self):
self.auth = FakeSession(token=uuid.uuid4().hex,

4
setup.cfg
View File

@ -27,6 +27,10 @@ packages =
data_files =
etc/ = etc/*
[entry_points]
oslo.config.opts =
mixmatch = mixmatch.config:list_opts
[build_sphinx]
source-dir = doc/source
build-dir = doc/build