From 6efe00dbf3959ebfbb49045f357823da0b94c3e0 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 21 Sep 2014 12:16:20 -0700 Subject: [PATCH] Port in config reading from shade --- README.rst | 109 ++++++++++++++++++++-- os_client_config/__init__.py | 10 +- os_client_config/cloud_config.py | 20 ++++ os_client_config/config.py | 150 ++++++++++++++++++++++++++++++ os_client_config/defaults_dict.py | 27 ++++++ os_client_config/exceptions.py | 17 ++++ os_client_config/vendors.py | 25 +++++ setup.py | 2 +- 8 files changed, 346 insertions(+), 14 deletions(-) create mode 100644 os_client_config/cloud_config.py create mode 100644 os_client_config/config.py create mode 100644 os_client_config/defaults_dict.py create mode 100644 os_client_config/exceptions.py create mode 100644 os_client_config/vendors.py diff --git a/README.rst b/README.rst index cf2315d..6332f83 100644 --- a/README.rst +++ b/README.rst @@ -2,14 +2,111 @@ os-client-config =============================== -OpenStack Client Configuation Library +os-client-config is a library for collecting client configuration for +using an OpenStack cloud in a consistent and comprehensive manner. It +will find cloud config for as few as 1 cloud and as many as you want to +put in a config file. It will read environment variables and config files, +and it also contains some vendor specific default values so that you don't +have to know extra info to use OpenStack + +Environment Variables +--------------------- + +os-client-config honors all of the normal `OS_*` variables. It does not +provide backwards compatibility to service-specific variables such as +`NOVA_USERNAME`. + +If you have environment variables and no config files, os-client-config +will produce a cloud config object named "openstack" containing your +values from the environment. + +Service specific settings, like the nova service type, are set with the +default service type as a prefix. For instance, to set a special service_type +for trove (because you're using Rackspace) set: +:: + + export OS_DATABASE_SERVICE_TYPE=rax:database + +Config Files +------------ + +os-client-config will for a file called clouds.yaml in the following locations: +* Current Directory +* ~/.config/openstack +* /etc/openstack + +The keys are all of the keys you'd expect from `OS_*` - except lower case +and without the OS prefix. So, username is set with `username`. + +Service specific settings, like the nova service type, are set with the +default service type as a prefix. For instance, to set a special service_type +for trove (because you're using Rackspace) set: +:: + + database_service_type: 'rax:database' + +An example config file is probably helpful: +:: + + clouds: + mordred: + cloud: hp + username: mordred@inaugust.com + password: XXXXXXXXX + project_id: mordred@inaugust.com + region_name: region-b.geo-1 + monty: + auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 + username: monty.taylor@hp.com + password: XXXXXXXX + project_id: monty.taylor@hp.com-default-tenant + region_name: region-b.geo-1 + infra: + cloud: rackspace + username: openstackci + password: XXXXXXXX + project_id: 610275 + region_name: DFW,ORD,IAD + +You may note a few things. First, since auth_url settings are silly +and embarrasingly ugly, known cloud vendors are included and may be referrenced +by name. One of the benefits of that is that auth_url isn't the only thing +the vendor defaults contain. For instance, since Rackspace is broken and lists +`rax:database` as the service type for trove, os-client-config knows that +so that you don't have to. + +Also, region_name can be a list of regions. When you call get_all_clouds, +you'll get a cloud config object for each cloud/region combo. + +Usage +----- + +The simplest and least useful thing you can do is: +:: + + python -m os_client_config.config + +Which will print out whatever if finds for your config. If you want to use +it from python, which is much more likely what you want to do, things like: + +Get a named cloud. +:: + + import os_client_config + + cloud_config = os_client_config.OpenStackConfig().get_one_cloud( + 'hp', 'region-b.geo-1') + print(cloud_config.name, cloud_config.region, cloud_config.config) + +Or, get all of the clouds. +:: + import os_client_config + + cloud_config = os_client_config.OpenStackConfig().get_all_clouds() + for cloud in cloud_config: + print(cloud.name, cloud.region, cloud.config) * Free software: Apache license * Documentation: http://docs.openstack.org/developer/os-client-config * Source: http://git.openstack.org/cgit/openstack/os-client-config * Bugs: http://bugs.launchpad.net/os-client-config - -Features --------- - -* TODO \ No newline at end of file diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index 26bdf7e..d5fd36c 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -1,5 +1,5 @@ -# -*- coding: utf-8 -*- - +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -12,8 +12,4 @@ # License for the specific language governing permissions and limitations # under the License. -import pbr.version - - -__version__ = pbr.version.VersionInfo( - 'os_client_config').version_string() \ No newline at end of file +from os_client_config.config import OpenStackConfig # noqa diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py new file mode 100644 index 0000000..d0c932f --- /dev/null +++ b/os_client_config/cloud_config.py @@ -0,0 +1,20 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class CloudConfig(object): + def __init__(self, name, region, config): + self.name = name + self.region = region + self.config = config diff --git a/os_client_config/config.py b/os_client_config/config.py new file mode 100644 index 0000000..1742b5b --- /dev/null +++ b/os_client_config/config.py @@ -0,0 +1,150 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import os + +import yaml + +from os_client_config import cloud_config +from os_client_config import defaults_dict +from os_client_config import exceptions +from os_client_config import vendors + +CONFIG_HOME = os.path.join(os.path.expanduser( + os.environ.get('XDG_CONFIG_HOME', os.path.join('~', '.config'))), + 'openstack') +CONFIG_SEARCH_PATH = [os.getcwd(), CONFIG_HOME, '/etc/openstack'] +CONFIG_FILES = [ + os.path.join(d, 'clouds.yaml') for d in CONFIG_SEARCH_PATH] +BOOL_KEYS = ('insecure', 'cache') +REQUIRED_VALUES = ('auth_url', 'username', 'password', 'project_id') +SERVICES = ( + 'compute', 'identity', 'network', 'metering', 'object-store', + 'volume', 'dns', 'image', 'database') + + +def get_boolean(value): + if value.lower() == 'true': + return True + return False + + +class OpenStackConfig(object): + + def __init__(self, config_files=None): + self._config_files = config_files or CONFIG_FILES + + defaults = defaults_dict.DefaultsDict() + defaults.add('username') + defaults.add('user_domain_name') + defaults.add('password') + defaults.add('project_id', defaults['username'], also='tenant_name') + defaults.add('project_domain_name') + defaults.add('auth_url') + defaults.add('region_name') + defaults.add('cache', 'false') + defaults.add('auth_token') + defaults.add('insecure', 'false') + defaults.add('cacert') + + for service in SERVICES: + defaults.add('service_name', prefix=service) + defaults.add('service_type', prefix=service) + defaults.add('endpoint_type', prefix=service) + defaults.add('endpoint', prefix=service) + self.defaults = defaults + + # use a config file if it exists where expected + self.cloud_config = self._load_config_file() + if not self.cloud_config: + self.cloud_config = dict( + clouds=dict(openstack=dict(self.defaults))) + + @classmethod + def get_services(klass): + return SERVICES + + def _load_config_file(self): + for path in self._config_files: + if os.path.exists(path): + return yaml.load(open(path, 'r')) + + def _get_regions(self, cloud): + return self.cloud_config['clouds'][cloud]['region_name'] + + def _get_region(self, cloud): + return self._get_regions(cloud).split(',')[0] + + def _get_cloud_sections(self): + return self.cloud_config['clouds'].keys() + + def _get_base_cloud_config(self, name): + cloud = dict() + if name in self.cloud_config['clouds']: + our_cloud = self.cloud_config['clouds'][name] + else: + our_cloud = dict() + + # yes, I know the next line looks silly + if 'cloud' in our_cloud: + cloud.update(vendors.CLOUD_DEFAULTS[our_cloud['cloud']]) + + cloud.update(self.defaults) + cloud.update(our_cloud) + if 'cloud' in cloud: + del cloud['cloud'] + return cloud + + def get_all_clouds(self): + + clouds = [] + + for cloud in self._get_cloud_sections(): + for region in self._get_regions(cloud).split(','): + clouds.append(self.get_one_cloud(cloud, region)) + return clouds + + def get_one_cloud(self, name='openstack', region=None): + + if not region: + region = self._get_region(name) + + config = self._get_base_cloud_config(name) + config['region_name'] = region + + for key in BOOL_KEYS: + if key in config: + config[key] = get_boolean(config[key]) + + for key in REQUIRED_VALUES: + if key not in config or not config[key]: + raise exceptions.OpenStackConfigException( + 'Unable to find full auth information for cloud {name} in' + ' config files {files}' + ' or environment variables.'.format( + name=name, files=','.join(self._config_files))) + + # If any of the defaults reference other values, we need to expand + for (key, value) in config.items(): + if hasattr(value, 'format'): + config[key] = value.format(**config) + + return cloud_config.CloudConfig( + name=name, region=region, config=config) + +if __name__ == '__main__': + config = OpenStackConfig().get_all_clouds() + for cloud in config: + print(cloud.name, cloud.region, cloud.config) diff --git a/os_client_config/defaults_dict.py b/os_client_config/defaults_dict.py new file mode 100644 index 0000000..43de77b --- /dev/null +++ b/os_client_config/defaults_dict.py @@ -0,0 +1,27 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + + +class DefaultsDict(dict): + + def add(self, key, default_value=None, also=None, prefix=None): + if prefix: + key = '%s_%s' % (prefix.replace('-', '_'), key) + if also: + value = os.environ.get(also, default_value) + value = os.environ.get('OS_%s' % key.upper(), default_value) + if value is not None: + self.__setitem__(key, value) diff --git a/os_client_config/exceptions.py b/os_client_config/exceptions.py new file mode 100644 index 0000000..ab78dc2 --- /dev/null +++ b/os_client_config/exceptions.py @@ -0,0 +1,17 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class OpenStackConfigException(Exception): + """Something went wrong with parsing your OpenStack Config.""" diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py new file mode 100644 index 0000000..d1b29a5 --- /dev/null +++ b/os_client_config/vendors.py @@ -0,0 +1,25 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +CLOUD_DEFAULTS = dict( + hp=dict( + auth_url='https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0', + region_name='region-b.geo-1', + ), + rackspace=dict( + auth_url='https://identity.api.rackspacecloud.com/v2.0/', + image_endpoint='https://{region_name}.images.api.rackspacecloud.com/', + database_service_type='rax:database', + ) +) diff --git a/setup.py b/setup.py index 7eeb36b..70c2b3f 100755 --- a/setup.py +++ b/setup.py @@ -19,4 +19,4 @@ import setuptools setuptools.setup( setup_requires=['pbr'], - pbr=True) \ No newline at end of file + pbr=True)