openstack
/
congress
Code Issues Proposed changes

Merge "Loads all available drivers by default"

This commit is contained in:
Zuul 2018-04-10 06:47:02 +00:00 committed by Gerrit Code Review
parent b822a530bf 82a708abcf
commit ada4038ea1
9 changed files with 156 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
congress/api/system/driver_model.py
View File

@ -40,7 +40,7 @@ class DatasourceDriverModel(base.APIModel):
drivers = self.bus.get_drivers_info()
fields = ['id', 'description']
results = [self.bus.make_datasource_dict(
drivers[driver], fields=fields)
driver, fields=fields)
for driver in drivers]
return {"results": results}

2
congress/common/config.py
View File

@ -52,6 +52,8 @@ core_opts = [
help=_('The type of authentication to use')),
cfg.ListOpt('drivers',
default=[],
deprecated_for_removal=True,
deprecated_reason='automatically loads all configured drivers',
help=_('List of driver class paths to import.')),
cfg.IntOpt('datasource_sync_period', default=60,
help='The number of seconds to wait between synchronizing '

131
congress/dse2/dse_node.py
View File

@ -22,9 +22,9 @@ from oslo_log import log as logging
import oslo_messaging as messaging
from oslo_messaging import exceptions as messaging_exceptions
from oslo_messaging.rpc import dispatcher
from oslo_utils import importutils
from oslo_utils import strutils
from oslo_utils import uuidutils
import stevedore
from congress.datalog import compile as datalog_compile
from congress.datasources import constants
@ -56,6 +56,8 @@ class DseNode(object):
CONTROL_TOPIC = 'congress-control'
SERVICE_TOPIC_PREFIX = 'congress-service-'
loaded_drivers = {}
def node_rpc_target(self, namespace=None, server=None, fanout=False):
return messaging.Target(exchange=self.EXCHANGE,
topic=self._add_partition(self.CONTROL_TOPIC),
@ -115,7 +117,7 @@ class DseNode(object):
self._control_bus = control_bus.DseNodeControlBus(self)
self.register_service(self._control_bus)
# load configured drivers
self.loaded_drivers = self.load_drivers()
# self.loaded_drivers = self.load_drivers()
self.periodic_tasks = None
self.sync_thread = None
self.start()
@ -503,36 +505,35 @@ class DseNode(object):
s._published_tables_with_subscriber = tables_with_subs
# Driver CRUD. Maybe belongs in a subclass of DseNode?
# Note(thread-safety): blocking function?
def load_drivers(self):
"""Load all configured drivers and check no name conflict"""
@classmethod
def load_drivers(cls):
"""Loads all configured drivers"""
result = {}
for driver_path in cfg.CONF.drivers:
# Note(thread-safety): blocking call?
obj = importutils.import_class(driver_path)
driver = obj.get_datasource_info()
if driver['id'] in result:
raise exception.BadConfig(_("There is a driver loaded already"
"with the driver name of %s")
% driver['id'])
driver['module'] = driver_path
result[driver['id']] = driver
return result
mgr = stevedore.extension.ExtensionManager(
namespace='congress.datasource.drivers',
invoke_on_load=False)
def get_driver_info(self, driver_name):
driver = self.loaded_drivers.get(driver_name)
for driver in mgr:
result[driver.name] = driver
cls.loaded_drivers = result
@classmethod
def get_driver_info(cls, driver_name):
driver = cls.loaded_drivers.get(driver_name)
if not driver:
raise exception.DriverNotFound(id=driver_name)
return driver
return driver.plugin.get_datasource_info()
def get_drivers_info(self):
return self.loaded_drivers
@classmethod
def get_drivers_info(cls):
drivers = cls.loaded_drivers.values()
return [d.plugin.get_datasource_info() for d in drivers]
def get_driver_schema(self, drivername):
driver = self.get_driver_info(drivername)
# Note(thread-safety): blocking call?
obj = importutils.import_class(driver['module'])
return obj.get_schema()
@classmethod
def get_driver_schema(cls, drivername):
driver = cls.loaded_drivers.get(drivername)
return driver.plugin.get_schema()
# Datasource CRUD. Maybe belongs in a subclass of DseNode?
# Note(thread-safety): blocking function
@ -608,35 +609,37 @@ class DseNode(object):
raise exception.InvalidDatasourceName(value=name)
driver = req['driver']
config = req['config'] or {}
for loaded_driver in self.loaded_drivers.values():
if loaded_driver['id'] == driver:
specified_options = set(config.keys())
valid_options = set(loaded_driver['config'].keys())
# Check that all the specified options passed in are
# valid configuration options that the driver exposes.
invalid_options = specified_options - valid_options
if invalid_options:
raise exception.InvalidDriverOption(
invalid_options=invalid_options)
# check that all the required options are passed in
required_options = set(
[k for k, v in loaded_driver['config'].items()
if v == constants.REQUIRED])
missing_options = required_options - specified_options
if ('project_name' in missing_options and
'tenant_name' in specified_options):
LOG.warning("tenant_name is deprecated, use project_name "
"instead")
missing_options.remove('project_name')
if missing_options:
missing_options = ', '.join(missing_options)
raise exception.MissingRequiredConfigOptions(
missing_options=missing_options)
return loaded_driver
try:
loaded_driver = self.get_driver_info(driver)
except exception.DriverNotFound:
raise exception.InvalidDriver(driver=req)
# If we get here no datasource driver match was found.
raise exception.InvalidDriver(driver=req)
specified_options = set(config.keys())
valid_options = set(loaded_driver['config'].keys())
# Check that all the specified options passed in are
# valid configuration options that the driver exposes.
invalid_options = specified_options - valid_options
if invalid_options:
raise exception.InvalidDriverOption(
invalid_options=invalid_options)
# check that all the required options are passed in
required_options = set(
[k for k, v in loaded_driver['config'].items()
if v == constants.REQUIRED])
missing_options = required_options - specified_options
if ('project_name' in missing_options and 'tenant_name' in
specified_options):
LOG.warning("tenant_name is deprecated, use project_name instead")
missing_options.remove('project_name')
if missing_options:
missing_options = ', '.join(missing_options)
raise exception.MissingRequiredConfigOptions(
missing_options=missing_options)
return loaded_driver
# Note (thread-safety): blocking function
def create_datasource_service(self, datasource):
@ -657,13 +660,9 @@ class DseNode(object):
LOG.info("datasource %s not enabled, skip loading",
ds_dict['name'])
return
driver_info = self.get_driver_info(ds_dict['driver'])
# split class_path into module and class name
class_path = driver_info['module']
pieces = class_path.split(".")
module_name = ".".join(pieces[:-1])
class_name = pieces[-1]
driver = self.loaded_drivers.get(ds_dict['driver'])
if not driver:
raise exception.DriverNotFound(id=ds_dict['driver'])
if ds_dict['config'] is None:
args = {'ds_id': ds_dict['id']}
@ -671,18 +670,14 @@ class DseNode(object):
args = dict(ds_dict['config'], ds_id=ds_dict['id'])
kwargs = {'name': ds_dict['name'], 'args': args}
LOG.info("creating service %s with class %s and args %s",
ds_dict['name'], module_name,
ds_dict['name'], driver.plugin,
strutils.mask_password(kwargs, "****"))
# import the module
try:
# Note(thread-safety): blocking call?
module = importutils.import_module(module_name)
service = getattr(module, class_name)(**kwargs)
service = driver.plugin(**kwargs)
except Exception:
msg = ("Error loading instance of module '%s'")
LOG.exception(msg, class_path)
raise exception.DataServiceError(msg % class_path)
LOG.exception(msg, driver.plugin)
raise exception.DataServiceError(msg % driver.plugin)
return service

3
congress/harness.py
View File

@ -73,6 +73,9 @@ def create2(node_id=None, bus_id=None, existing_node=None,
# create services as required
services = {}
# Load all configured drivers
dse_node.DseNode.load_drivers()
if datasources:
LOG.info("Registering congress datasource services on node %s",
node.node_id)

16
congress/tests/api/test_driver_model.py
View File

@ -20,6 +20,7 @@ from __future__ import absolute_import
from congress.api import webservice
from congress.tests.api import base as api_base
from congress.tests import base
from congress.tests import helper
class TestDriverModel(base.SqlTestCase):
@ -46,15 +47,11 @@ class TestDriverModel(base.SqlTestCase):
def test_drivers_list(self):
context = {}
expected_ret = {"results": [
{
"description": "This is a fake driver used for testing",
"id": "fake_datasource"
}
]}
ret = self.driver_model.get_items({}, context)
self.assertEqual(expected_ret, ret)
drivers = helper.supported_drivers()
expected_ret = sorted(drivers, key=lambda d: d['id'])
ret = self.driver_model.get_items({}, context)['results']
actual_ret = sorted(ret, key=lambda d: d['id'])
self.assertEqual(expected_ret, actual_ret)
def test_driver_details(self):
context = {
@ -75,7 +72,6 @@ class TestDriverModel(base.SqlTestCase):
},
"description": "This is a fake driver used for testing",
"id": "fake_datasource",
"module": "congress.tests.fake_datasource.FakeDataSource",
"secret": ["password"],
"tables": [{'columns': [
{'description': None, 'name': 'id'},

10
congress/tests/dse2/test_datasource.py
View File

@ -18,7 +18,6 @@ from __future__ import division
from __future__ import absolute_import
import mock
from oslo_config import cfg
from oslo_db import exception as db_exc
from congress.db import datasources as datasource_db
@ -194,12 +193,3 @@ class TestDataSource(base.SqlTestCase):
# self.assertEqual(
# schema,
# fake_datasource.FakeDataSource.get_schema())
def test_duplicate_driver_name_raises(self):
# Load the driver twice
cfg.CONF.set_override(
'drivers',
['congress.tests.fake_datasource.FakeDataSource',
'congress.tests.fake_datasource.FakeDataSource'])
self.assertRaises(congressException.BadConfig,
self.dseNode.load_drivers)

6
congress/tests/dse2/test_dse_node.py
View File

@ -315,14 +315,14 @@ class TestDseNode(base.SqlTestCase):
'password': '<hidden>',
'tenant_name': 'armax'}}
@mock.patch.object(dse_node.DseNode, 'validate_create_datasource')
@mock.patch.object(dse_node.DseNode, 'get_driver_info')
def test_missing_driver_datasources(self, mock_driver_info):
def test_missing_driver_datasources(self, mock_driver_info, mock_validate):
services = api_base.setup_config(api=False, policy=False)
node = services['node']
ds_manager = services['ds_manager']
ds = self._get_datasource_request()
mock_driver_info.return_value = {'secret': [],
'module': mock.MagicMock()}
mock_driver_info.return_value = {'secret': []}
ds_manager.add_datasource(ds)
mock_driver_info.side_effect = [exception.DriverNotFound]
node.delete_missing_driver_datasources()

56
congress/tests/helper.py
View File

@ -458,3 +458,59 @@ class TestFailureException(Exception):
"""
def __init__(self, *args, **kwargs):
Exception.__init__(self, *args, **kwargs)
def supported_drivers():
"""Get list of supported drivers by congress"""
results = [
{"id": "monasca",
"description": "Datasource driver that interfaces with monasca."},
{"id": "plexxi",
"description": "Datasource driver that interfaces with PlexxiCore."},
{"id": "doctor",
"description": "Datasource driver that allows external systems "
"to push data in accordance with OPNFV Doctor "
"Inspector southbound interface specification."},
{"id": "aodh",
"description": "Datasource driver that interfaces with aodh."},
{"id": "neutronv2_qos",
"description": "Datasource driver that interfaces with QoS "
"extension of OpenStack Networking aka Neutron."},
{"id": "cloudfoundryv2",
"description": "Datasource driver that interfaces with cloudfoundry"},
{"id": "heat",
"description": "Datasource driver that interfaces with OpenStack "
"orchestration aka heat."},
{"id": "nova",
"description": "Datasource driver that interfaces with OpenStack "
"Compute aka nova."},
{"id": "murano",
"description": "Datasource driver that interfaces with murano"},
{"id": "neutronv2",
"description": "Datasource driver that interfaces with OpenStack "
"Networking aka Neutron."},
{"id": "swift",
"description": "Datasource driver that interfaces with swift."},
{"id": "ironic",
"description": "Datasource driver that interfaces with OpenStack "
"bare metal aka ironic."},
{"id": "cinder",
"description": "Datasource driver that interfaces with OpenStack "
"cinder."},
{"id": "fake_datasource",
"description": "This is a fake driver used for testing"},
{"id": "config",
"description": "Datasource driver that allows OS configs retrieval."},
{"id": "glancev2",
"description": "Datasource driver that interfaces with OpenStack "
"Images aka Glance."},
{"id": "vcenter",
"description": "Datasource driver that interfaces with vcenter"},
{"id": "keystonev3",
"description": "Datasource driver that interfaces with keystone."},
{"id": "keystone",
"description": "Datasource driver that interfaces with keystone."},
{"id": "mistral",
"description": "Datasource driver that interfaces with Mistral."}]
return results

22
setup.cfg
View File

@ -59,6 +59,28 @@ console_scripts =
congress-db-manage = congress.db.migration.cli:main
congress-cfg-validator-agt = congress.cfg_validator.agent.agent:main
congress.datasource.drivers =
aodh = congress.datasources.aodh_driver:AodhDriver
cinder = congress.datasources.cinder_driver:CinderDriver
cloudfoundryv2 = congress.datasources.cloudfoundryv2_driver:CloudFoundryV2Driver
config = congress.datasources.cfgvalidator_driver:ValidatorDriver
doctor = congress.datasources.doctor_driver:DoctorDriver
fake_datasource = congress.tests.fake_datasource:FakeDataSource
glancev2 = congress.datasources.glancev2_driver:GlanceV2Driver
heat = congress.datasources.heatv1_driver:HeatV1Driver
ironic = congress.datasources.ironic_driver:IronicDriver
keystone = congress.datasources.keystone_driver:KeystoneDriver
keystonev3 = congress.datasources.keystonev3_driver:KeystoneV3Driver
mistral = congress.datasources.mistral_driver:MistralDriver
monasca = congress.datasources.monasca_driver:MonascaDriver
murano = congress.datasources.murano_driver:MuranoDriver
neutronv2 = congress.datasources.neutronv2_driver:NeutronV2Driver
neutronv2_qos = congress.datasources.neutronv2_qos_driver:NeutronV2QosDriver
nova = congress.datasources.nova_driver:NovaDriver
plexxi = congress.datasources.plexxi_driver:PlexxiDriver
swift = congress.datasources.swift_driver:SwiftDriver
vcenter = congress.datasources.vCenter_driver:VCenterDriver
[build_sphinx]
all_files = 1
build-dir = doc/build