openstack
/
congress
Code Issues Proposed changes

Loads all available drivers by default 

This commit loads all datasources drivers by default without the need
of config option. The config option 'drivers' would be deprecated as it
is required to update manually on newly supported drivers for each release
and also error prone.

To disable any of the drivers, can be disabled using the config option
'disabled_drivers'

For example,
disabled_drivers = plexxi, murano

TODO: Support disabled_drivers option to disable the datasource drivers

Partially-Implements blueprint enable-drivers-by-default
Change-Id: I5f9878d07c2e1487f8f14f7cd1948560327d8083
This commit is contained in:
Anusha Ramineni 2018-01-16 13:01:31 +05:30
parent ff2aab3fec
commit 82a708abcf
9 changed files with 156 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
congress/api/system/driver_model.py
View File

@ -40,7 +40,7 @@ class DatasourceDriverModel(base.APIModel):
drivers = self.bus.get_drivers_info()
fields = ['id', 'description']
results = [self.bus.make_datasource_dict(
drivers[driver], fields=fields)
driver, fields=fields)
for driver in drivers]
return {"results": results}

2
congress/common/config.py
View File

@ -52,6 +52,8 @@ core_opts = [
help=_('The type of authentication to use')),
cfg.ListOpt('drivers',
default=[],
deprecated_for_removal=True,
deprecated_reason='automatically loads all configured drivers',
help=_('List of driver class paths to import.')),
cfg.IntOpt('datasource_sync_period', default=60,
help='The number of seconds to wait between synchronizing '

131
congress/dse2/dse_node.py
View File

@ -22,9 +22,9 @@ from oslo_log import log as logging
import oslo_messaging as messaging
from oslo_messaging import exceptions as messaging_exceptions
from oslo_messaging.rpc import dispatcher
from oslo_utils import importutils
from oslo_utils import strutils
from oslo_utils import uuidutils
import stevedore
from congress.datalog import compile as datalog_compile
from congress.datasources import constants
@ -56,6 +56,8 @@ class DseNode(object):
CONTROL_TOPIC = 'congress-control'
SERVICE_TOPIC_PREFIX = 'congress-service-'
loaded_drivers = {}
def node_rpc_target(self, namespace=None, server=None, fanout=False):
return messaging.Target(exchange=self.EXCHANGE,
topic=self._add_partition(self.CONTROL_TOPIC),
@ -119,7 +121,7 @@ class DseNode(object):
self._control_bus = control_bus.DseNodeControlBus(self)
self.register_service(self._control_bus)
# load configured drivers
self.loaded_drivers = self.load_drivers()
# self.loaded_drivers = self.load_drivers()
self.periodic_tasks = None
self.sync_thread = None
self.start()
@ -513,36 +515,35 @@ class DseNode(object):
s._published_tables_with_subscriber = tables_with_subs
# Driver CRUD. Maybe belongs in a subclass of DseNode?
# Note(thread-safety): blocking function?
def load_drivers(self):
"""Load all configured drivers and check no name conflict"""
@classmethod
def load_drivers(cls):
"""Loads all configured drivers"""
result = {}
for driver_path in cfg.CONF.drivers:
# Note(thread-safety): blocking call?
obj = importutils.import_class(driver_path)
driver = obj.get_datasource_info()
if driver['id'] in result:
raise exception.BadConfig(_("There is a driver loaded already"
"with the driver name of %s")
% driver['id'])
driver['module'] = driver_path
result[driver['id']] = driver
return result
mgr = stevedore.extension.ExtensionManager(
namespace='congress.datasource.drivers',
invoke_on_load=False)
def get_driver_info(self, driver_name):
driver = self.loaded_drivers.get(driver_name)
for driver in mgr:
result[driver.name] = driver
cls.loaded_drivers = result
@classmethod
def get_driver_info(cls, driver_name):
driver = cls.loaded_drivers.get(driver_name)
if not driver:
raise exception.DriverNotFound(id=driver_name)
return driver
return driver.plugin.get_datasource_info()
def get_drivers_info(self):
return self.loaded_drivers
@classmethod
def get_drivers_info(cls):
drivers = cls.loaded_drivers.values()
return [d.plugin.get_datasource_info() for d in drivers]
def get_driver_schema(self, drivername):
driver = self.get_driver_info(drivername)
# Note(thread-safety): blocking call?
obj = importutils.import_class(driver['module'])
return obj.get_schema()
@classmethod
def get_driver_schema(cls, drivername):
driver = cls.loaded_drivers.get(drivername)
return driver.plugin.get_schema()
# Datasource CRUD. Maybe belongs in a subclass of DseNode?
# Note(thread-safety): blocking function
@ -618,35 +619,37 @@ class DseNode(object):
raise exception.InvalidDatasourceName(value=name)
driver = req['driver']
config = req['config'] or {}
for loaded_driver in self.loaded_drivers.values():
if loaded_driver['id'] == driver:
specified_options = set(config.keys())
valid_options = set(loaded_driver['config'].keys())
# Check that all the specified options passed in are
# valid configuration options that the driver exposes.
invalid_options = specified_options - valid_options
if invalid_options:
raise exception.InvalidDriverOption(
invalid_options=invalid_options)
# check that all the required options are passed in
required_options = set(
[k for k, v in loaded_driver['config'].items()
if v == constants.REQUIRED])
missing_options = required_options - specified_options
if ('project_name' in missing_options and
'tenant_name' in specified_options):
LOG.warning("tenant_name is deprecated, use project_name "
"instead")
missing_options.remove('project_name')
if missing_options:
missing_options = ', '.join(missing_options)
raise exception.MissingRequiredConfigOptions(
missing_options=missing_options)
return loaded_driver
try:
loaded_driver = self.get_driver_info(driver)
except exception.DriverNotFound:
raise exception.InvalidDriver(driver=req)
# If we get here no datasource driver match was found.
raise exception.InvalidDriver(driver=req)
specified_options = set(config.keys())
valid_options = set(loaded_driver['config'].keys())
# Check that all the specified options passed in are
# valid configuration options that the driver exposes.
invalid_options = specified_options - valid_options
if invalid_options:
raise exception.InvalidDriverOption(
invalid_options=invalid_options)
# check that all the required options are passed in
required_options = set(
[k for k, v in loaded_driver['config'].items()
if v == constants.REQUIRED])
missing_options = required_options - specified_options
if ('project_name' in missing_options and 'tenant_name' in
specified_options):
LOG.warning("tenant_name is deprecated, use project_name instead")
missing_options.remove('project_name')
if missing_options:
missing_options = ', '.join(missing_options)
raise exception.MissingRequiredConfigOptions(
missing_options=missing_options)
return loaded_driver
# Note (thread-safety): blocking function
def create_datasource_service(self, datasource):
@ -667,13 +670,9 @@ class DseNode(object):
LOG.info("datasource %s not enabled, skip loading",
ds_dict['name'])
return
driver_info = self.get_driver_info(ds_dict['driver'])
# split class_path into module and class name
class_path = driver_info['module']
pieces = class_path.split(".")
module_name = ".".join(pieces[:-1])
class_name = pieces[-1]
driver = self.loaded_drivers.get(ds_dict['driver'])
if not driver:
raise exception.DriverNotFound(id=ds_dict['driver'])
if ds_dict['config'] is None:
args = {'ds_id': ds_dict['id']}
@ -681,18 +680,14 @@ class DseNode(object):
args = dict(ds_dict['config'], ds_id=ds_dict['id'])
kwargs = {'name': ds_dict['name'], 'args': args}
LOG.info("creating service %s with class %s and args %s",
ds_dict['name'], module_name,
ds_dict['name'], driver.plugin,
strutils.mask_password(kwargs, "****"))
# import the module
try:
# Note(thread-safety): blocking call?
module = importutils.import_module(module_name)
service = getattr(module, class_name)(**kwargs)
service = driver.plugin(**kwargs)
except Exception:
msg = ("Error loading instance of module '%s'")
LOG.exception(msg, class_path)
raise exception.DataServiceError(msg % class_path)
LOG.exception(msg, driver.plugin)
raise exception.DataServiceError(msg % driver.plugin)
return service

3
congress/harness.py
View File

@ -73,6 +73,9 @@ def create2(node_id=None, bus_id=None, existing_node=None,
# create services as required
services = {}
# Load all configured drivers
dse_node.DseNode.load_drivers()
if datasources:
LOG.info("Registering congress datasource services on node %s",
node.node_id)

16
congress/tests/api/test_driver_model.py
View File

@ -20,6 +20,7 @@ from __future__ import absolute_import
from congress.api import webservice
from congress.tests.api import base as api_base
from congress.tests import base
from congress.tests import helper
class TestDriverModel(base.SqlTestCase):
@ -46,15 +47,11 @@ class TestDriverModel(base.SqlTestCase):
def test_drivers_list(self):
context = {}
expected_ret = {"results": [
{
"description": "This is a fake driver used for testing",
"id": "fake_datasource"
}
]}
ret = self.driver_model.get_items({}, context)
self.assertEqual(expected_ret, ret)
drivers = helper.supported_drivers()
expected_ret = sorted(drivers, key=lambda d: d['id'])
ret = self.driver_model.get_items({}, context)['results']
actual_ret = sorted(ret, key=lambda d: d['id'])
self.assertEqual(expected_ret, actual_ret)
def test_driver_details(self):
context = {
@ -75,7 +72,6 @@ class TestDriverModel(base.SqlTestCase):
},
"description": "This is a fake driver used for testing",
"id": "fake_datasource",
"module": "congress.tests.fake_datasource.FakeDataSource",
"secret": ["password"],
"tables": [{'columns': [
{'description': None, 'name': 'id'},

10
congress/tests/dse2/test_datasource.py
View File

@ -18,7 +18,6 @@ from __future__ import division
from __future__ import absolute_import
import mock
from oslo_config import cfg
from oslo_db import exception as db_exc
from congress.db import datasources as datasource_db
@ -194,12 +193,3 @@ class TestDataSource(base.SqlTestCase):
# self.assertEqual(
# schema,
# fake_datasource.FakeDataSource.get_schema())
def test_duplicate_driver_name_raises(self):
# Load the driver twice
cfg.CONF.set_override(
'drivers',
['congress.tests.fake_datasource.FakeDataSource',
'congress.tests.fake_datasource.FakeDataSource'])
self.assertRaises(congressException.BadConfig,
self.dseNode.load_drivers)

6
congress/tests/dse2/test_dse_node.py
View File

@ -315,14 +315,14 @@ class TestDseNode(base.SqlTestCase):
'password': '<hidden>',
'tenant_name': 'armax'}}
@mock.patch.object(dse_node.DseNode, 'validate_create_datasource')
@mock.patch.object(dse_node.DseNode, 'get_driver_info')
def test_missing_driver_datasources(self, mock_driver_info):
def test_missing_driver_datasources(self, mock_driver_info, mock_validate):
services = api_base.setup_config(api=False, policy=False)
node = services['node']
ds_manager = services['ds_manager']
ds = self._get_datasource_request()
mock_driver_info.return_value = {'secret': [],
'module': mock.MagicMock()}
mock_driver_info.return_value = {'secret': []}
ds_manager.add_datasource(ds)
mock_driver_info.side_effect = [exception.DriverNotFound]
node.delete_missing_driver_datasources()

56
congress/tests/helper.py
View File

@ -458,3 +458,59 @@ class TestFailureException(Exception):
"""
def __init__(self, *args, **kwargs):
Exception.__init__(self, *args, **kwargs)
def supported_drivers():
"""Get list of supported drivers by congress"""
results = [
{"id": "monasca",
"description": "Datasource driver that interfaces with monasca."},
{"id": "plexxi",
"description": "Datasource driver that interfaces with PlexxiCore."},
{"id": "doctor",
"description": "Datasource driver that allows external systems "
"to push data in accordance with OPNFV Doctor "
"Inspector southbound interface specification."},
{"id": "aodh",
"description": "Datasource driver that interfaces with aodh."},
{"id": "neutronv2_qos",
"description": "Datasource driver that interfaces with QoS "
"extension of OpenStack Networking aka Neutron."},
{"id": "cloudfoundryv2",
"description": "Datasource driver that interfaces with cloudfoundry"},
{"id": "heat",
"description": "Datasource driver that interfaces with OpenStack "
"orchestration aka heat."},
{"id": "nova",
"description": "Datasource driver that interfaces with OpenStack "
"Compute aka nova."},
{"id": "murano",
"description": "Datasource driver that interfaces with murano"},
{"id": "neutronv2",
"description": "Datasource driver that interfaces with OpenStack "
"Networking aka Neutron."},
{"id": "swift",
"description": "Datasource driver that interfaces with swift."},
{"id": "ironic",
"description": "Datasource driver that interfaces with OpenStack "
"bare metal aka ironic."},
{"id": "cinder",
"description": "Datasource driver that interfaces with OpenStack "
"cinder."},
{"id": "fake_datasource",
"description": "This is a fake driver used for testing"},
{"id": "config",
"description": "Datasource driver that allows OS configs retrieval."},
{"id": "glancev2",
"description": "Datasource driver that interfaces with OpenStack "
"Images aka Glance."},
{"id": "vcenter",
"description": "Datasource driver that interfaces with vcenter"},
{"id": "keystonev3",
"description": "Datasource driver that interfaces with keystone."},
{"id": "keystone",
"description": "Datasource driver that interfaces with keystone."},
{"id": "mistral",
"description": "Datasource driver that interfaces with Mistral."}]
return results

22
setup.cfg
View File

@ -59,6 +59,28 @@ console_scripts =
congress-db-manage = congress.db.migration.cli:main
congress-cfg-validator-agt = congress.cfg_validator.agent.agent:main
congress.datasource.drivers =
aodh = congress.datasources.aodh_driver:AodhDriver
cinder = congress.datasources.cinder_driver:CinderDriver
cloudfoundryv2 = congress.datasources.cloudfoundryv2_driver:CloudFoundryV2Driver
config = congress.datasources.cfgvalidator_driver:ValidatorDriver
doctor = congress.datasources.doctor_driver:DoctorDriver
fake_datasource = congress.tests.fake_datasource:FakeDataSource
glancev2 = congress.datasources.glancev2_driver:GlanceV2Driver
heat = congress.datasources.heatv1_driver:HeatV1Driver
ironic = congress.datasources.ironic_driver:IronicDriver
keystone = congress.datasources.keystone_driver:KeystoneDriver
keystonev3 = congress.datasources.keystonev3_driver:KeystoneV3Driver
mistral = congress.datasources.mistral_driver:MistralDriver
monasca = congress.datasources.monasca_driver:MonascaDriver
murano = congress.datasources.murano_driver:MuranoDriver
neutronv2 = congress.datasources.neutronv2_driver:NeutronV2Driver
neutronv2_qos = congress.datasources.neutronv2_qos_driver:NeutronV2QosDriver
nova = congress.datasources.nova_driver:NovaDriver
plexxi = congress.datasources.plexxi_driver:PlexxiDriver
swift = congress.datasources.swift_driver:SwiftDriver
vcenter = congress.datasources.vCenter_driver:VCenterDriver
[build_sphinx]
all_files = 1
build-dir = doc/build