From 29aa644eb33cbf6bd54935fd0d0f3283872f72cc Mon Sep 17 00:00:00 2001 From: Doug Wiegley Date: Tue, 27 Jan 2015 20:17:00 -0700 Subject: [PATCH] Mechanisms to move extensions and config into service repos - Extensions will automatically be loaded from service repos in addition to neutron proper, but neutron proper will take precedence. - Config entries for service repos will be read out of neutron-{service}.conf first, and then neutron.conf. After Kilo, they will be read only from neutron-{service}.conf. - Service providers for drivers will be collected from all neutron conf files. This is review 1 of 3. The second set will be in the server repos, moving the extensions. The third will be in neutron, removing the service exts. Change-Id: I16b5e5b2bb70717166da14faa975fa2ab9129049 Partially-Implements: blueprint services-split --- etc/neutron.conf | 1 + neutron/api/extensions.py | 32 +++++++-- neutron/common/repos.py | 68 +++++++++++++++++++ neutron/services/provider_configuration.py | 31 ++++++++- .../tests/unit/test_provider_configuration.py | 17 +---- 5 files changed, 128 insertions(+), 21 deletions(-) create mode 100644 neutron/common/repos.py diff --git a/etc/neutron.conf b/etc/neutron.conf index 3c99c033a1f..97bc03da603 100644 --- a/etc/neutron.conf +++ b/etc/neutron.conf @@ -659,6 +659,7 @@ admin_password = %SERVICE_PASSWORD% # If set, use this value for pool_timeout with sqlalchemy # pool_timeout = 10 +# TODO(dougwig) - remove these lines once service repos have them [service_providers] # Specify service providers (drivers) for advanced services like loadbalancer, VPN, Firewall. # Must be in form: diff --git a/neutron/api/extensions.py b/neutron/api/extensions.py index 358067787cb..b1294e155ea 100644 --- a/neutron/api/extensions.py +++ b/neutron/api/extensions.py @@ -26,6 +26,7 @@ import webob.dec import webob.exc from neutron.common import exceptions +from neutron.common import repos import neutron.extensions from neutron.i18n import _LE, _LI, _LW from neutron import manager @@ -518,13 +519,19 @@ class ExtensionManager(object): See tests/unit/extensions/foxinsocks.py for an example extension implementation. """ + + # TODO(dougwig) - remove this after the service extensions move out + # While moving the extensions out of neutron into the service repos, + # don't double-load the same thing. + loaded = [] + for path in self.path.split(':'): if os.path.exists(path): - self._load_all_extensions_from_path(path) + self._load_all_extensions_from_path(path, loaded) else: LOG.error(_LE("Extension path '%s' doesn't exist!"), path) - def _load_all_extensions_from_path(self, path): + def _load_all_extensions_from_path(self, path, loaded): # Sorting the extension list makes the order in which they # are loaded predictable across a cluster of load-balanced # Neutron Servers @@ -534,7 +541,12 @@ class ExtensionManager(object): mod_name, file_ext = os.path.splitext(os.path.split(f)[-1]) ext_path = os.path.join(path, f) if file_ext.lower() == '.py' and not mod_name.startswith('_'): + if mod_name in loaded: + LOG.warn(_LW("Extension already loaded, skipping: %s"), + mod_name) + continue mod = imp.load_source(mod_name, ext_path) + loaded.append(mod_name) ext_name = mod_name[0].upper() + mod_name[1:] new_ext_class = getattr(mod, ext_name, None) if not new_ext_class: @@ -661,11 +673,19 @@ class ResourceExtension(object): # Returns the extension paths from a config entry and the __path__ # of neutron.extensions def get_extensions_path(): - paths = ':'.join(neutron.extensions.__path__) - if cfg.CONF.api_extensions_path: - paths = ':'.join([cfg.CONF.api_extensions_path, paths]) + paths = neutron.extensions.__path__ - return paths + neutron_mods = repos.NeutronModules() + for x in neutron_mods.installed_list(): + paths += neutron_mods.module(x).__path__ + + if cfg.CONF.api_extensions_path: + paths.append(cfg.CONF.api_extensions_path) + + LOG.debug("get_extension_paths = %s", paths) + + path = ':'.join(paths) + return path def append_api_extensions_path(paths): diff --git a/neutron/common/repos.py b/neutron/common/repos.py new file mode 100644 index 00000000000..bf848c5a730 --- /dev/null +++ b/neutron/common/repos.py @@ -0,0 +1,68 @@ +# Copyright (c) 2015, A10 Networks +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ConfigParser +import importlib +import os + +from neutron.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + + +class NeutronModules(object): + + MODULES = [ + 'neutron_fwaas', + 'neutron_lbaas', + 'neutron_vpnaas' + ] + + def __init__(self): + self.repos = {} + for repo in self.MODULES: + self.repos[repo] = {} + self.repos[repo]['mod'] = self._import_or_none(repo) + self.repos[repo]['ini'] = None + + def _import_or_none(self, module): + try: + return importlib.import_module(module) + except ImportError: + return None + + def installed_list(self): + z = filter(lambda k: self.repos[k]['mod'] is not None, self.repos) + LOG.debug("NeutronModules related repos installed = %s", z) + return z + + def module(self, module): + return self.repos[module]['mod'] + + # Return an INI parser for the child module. oslo.conf is a bit too + # magical in its INI loading, and in one notable case, we need to merge + # together the [service_providers] section for across at least four + # repositories. + def ini(self, module): + if self.repos[module]['ini'] is None: + ini = ConfigParser.SafeConfigParser() + + ini_path = '/etc/neutron/%s.conf' % module + if os.path.exists(ini_path): + ini.read(ini_path) + + self.repos[module]['ini'] = ini + + return self.repos[module]['ini'] diff --git a/neutron/services/provider_configuration.py b/neutron/services/provider_configuration.py index 841954b018e..6bc0ed73059 100644 --- a/neutron/services/provider_configuration.py +++ b/neutron/services/provider_configuration.py @@ -17,6 +17,7 @@ from oslo.config import cfg import stevedore from neutron.common import exceptions as n_exc +from neutron.common import repos from neutron.i18n import _LW from neutron.openstack.common import log as logging from neutron.plugins.common import constants @@ -69,7 +70,35 @@ def parse_service_provider_opt(): raise n_exc.Invalid( _("Provider name is limited by 255 characters: %s") % name) - svc_providers_opt = cfg.CONF.service_providers.service_provider + # Main neutron config file + try: + svc_providers_opt = cfg.CONF.service_providers.service_provider + except cfg.NoSuchOptError: + svc_providers_opt = [] + + # Add in entries from the *aas conf files + neutron_mods = repos.NeutronModules() + for x in neutron_mods.installed_list(): + ini = neutron_mods.ini(x) + if ini is None: + continue + + try: + sp = ini.items('service_providers') + for name, value in sp: + if name == 'service_provider': + svc_providers_opt.append(value) + except Exception: + continue + + # TODO(dougwig) - remove this next bit after we've migrated all entries + # to the service repo config files. Some tests require a default driver + # to be present, but not two, which leads to a cross-repo breakage + # issue. uniq the list as a short-term workaround. + svc_providers_opt = list(set(svc_providers_opt)) + + LOG.debug("Service providers = %s", svc_providers_opt) + res = [] for prov_def in svc_providers_opt: split = prov_def.split(':') diff --git a/neutron/tests/unit/test_provider_configuration.py b/neutron/tests/unit/test_provider_configuration.py index 8a6378a06a3..89233b9f1d7 100644 --- a/neutron/tests/unit/test_provider_configuration.py +++ b/neutron/tests/unit/test_provider_configuration.py @@ -60,21 +60,10 @@ class ParseServiceProviderConfigurationTestCase(base.BaseTestCase): constants.LOADBALANCER + ':name2:path2:default'], 'service_providers') - expected = {'service_type': constants.LOADBALANCER, - 'name': 'lbaas', - 'driver': 'driver_path', - 'default': False} res = provconf.parse_service_provider_opt() - self.assertEqual(len(res), 3) - self.assertEqual(res, [expected, - {'service_type': constants.LOADBALANCER, - 'name': 'name1', - 'driver': 'path1', - 'default': False}, - {'service_type': constants.LOADBALANCER, - 'name': 'name2', - 'driver': 'path2', - 'default': True}]) + # This parsing crosses repos if additional projects are installed, + # so check that at least what we expect is there; there may be more. + self.assertTrue(len(res) >= 3) def test_parse_service_provider_opt_not_allowed_raises(self): cfg.CONF.set_override('service_provider',