diff --git a/congress/api/system/driver_model.py b/congress/api/system/driver_model.py index 0161419a4..27ec589eb 100644 --- a/congress/api/system/driver_model.py +++ b/congress/api/system/driver_model.py @@ -40,7 +40,7 @@ class DatasourceDriverModel(base.APIModel): drivers = self.bus.get_drivers_info() fields = ['id', 'description'] results = [self.bus.make_datasource_dict( - drivers[driver], fields=fields) + driver, fields=fields) for driver in drivers] return {"results": results} diff --git a/congress/common/config.py b/congress/common/config.py index 72adae438..5bf62fceb 100644 --- a/congress/common/config.py +++ b/congress/common/config.py @@ -52,6 +52,8 @@ core_opts = [ help=_('The type of authentication to use')), cfg.ListOpt('drivers', default=[], + deprecated_for_removal=True, + deprecated_reason='automatically loads all configured drivers', help=_('List of driver class paths to import.')), cfg.IntOpt('datasource_sync_period', default=60, help='The number of seconds to wait between synchronizing ' diff --git a/congress/dse2/dse_node.py b/congress/dse2/dse_node.py index 5b1e7fbb2..e10c7ac67 100644 --- a/congress/dse2/dse_node.py +++ b/congress/dse2/dse_node.py @@ -22,9 +22,9 @@ from oslo_log import log as logging import oslo_messaging as messaging from oslo_messaging import exceptions as messaging_exceptions from oslo_messaging.rpc import dispatcher -from oslo_utils import importutils from oslo_utils import strutils from oslo_utils import uuidutils +import stevedore from congress.datalog import compile as datalog_compile from congress.datasources import constants @@ -56,6 +56,8 @@ class DseNode(object): CONTROL_TOPIC = 'congress-control' SERVICE_TOPIC_PREFIX = 'congress-service-' + loaded_drivers = {} + def node_rpc_target(self, namespace=None, server=None, fanout=False): return messaging.Target(exchange=self.EXCHANGE, topic=self._add_partition(self.CONTROL_TOPIC), @@ -115,7 +117,7 @@ class DseNode(object): self._control_bus = control_bus.DseNodeControlBus(self) self.register_service(self._control_bus) # load configured drivers - self.loaded_drivers = self.load_drivers() + # self.loaded_drivers = self.load_drivers() self.periodic_tasks = None self.sync_thread = None self.start() @@ -503,36 +505,35 @@ class DseNode(object): s._published_tables_with_subscriber = tables_with_subs # Driver CRUD. Maybe belongs in a subclass of DseNode? - # Note(thread-safety): blocking function? - def load_drivers(self): - """Load all configured drivers and check no name conflict""" + @classmethod + def load_drivers(cls): + """Loads all configured drivers""" result = {} - for driver_path in cfg.CONF.drivers: - # Note(thread-safety): blocking call? - obj = importutils.import_class(driver_path) - driver = obj.get_datasource_info() - if driver['id'] in result: - raise exception.BadConfig(_("There is a driver loaded already" - "with the driver name of %s") - % driver['id']) - driver['module'] = driver_path - result[driver['id']] = driver - return result + mgr = stevedore.extension.ExtensionManager( + namespace='congress.datasource.drivers', + invoke_on_load=False) - def get_driver_info(self, driver_name): - driver = self.loaded_drivers.get(driver_name) + for driver in mgr: + result[driver.name] = driver + + cls.loaded_drivers = result + + @classmethod + def get_driver_info(cls, driver_name): + driver = cls.loaded_drivers.get(driver_name) if not driver: raise exception.DriverNotFound(id=driver_name) - return driver + return driver.plugin.get_datasource_info() - def get_drivers_info(self): - return self.loaded_drivers + @classmethod + def get_drivers_info(cls): + drivers = cls.loaded_drivers.values() + return [d.plugin.get_datasource_info() for d in drivers] - def get_driver_schema(self, drivername): - driver = self.get_driver_info(drivername) - # Note(thread-safety): blocking call? - obj = importutils.import_class(driver['module']) - return obj.get_schema() + @classmethod + def get_driver_schema(cls, drivername): + driver = cls.loaded_drivers.get(drivername) + return driver.plugin.get_schema() # Datasource CRUD. Maybe belongs in a subclass of DseNode? # Note(thread-safety): blocking function @@ -608,35 +609,37 @@ class DseNode(object): raise exception.InvalidDatasourceName(value=name) driver = req['driver'] config = req['config'] or {} - for loaded_driver in self.loaded_drivers.values(): - if loaded_driver['id'] == driver: - specified_options = set(config.keys()) - valid_options = set(loaded_driver['config'].keys()) - # Check that all the specified options passed in are - # valid configuration options that the driver exposes. - invalid_options = specified_options - valid_options - if invalid_options: - raise exception.InvalidDriverOption( - invalid_options=invalid_options) - # check that all the required options are passed in - required_options = set( - [k for k, v in loaded_driver['config'].items() - if v == constants.REQUIRED]) - missing_options = required_options - specified_options - if ('project_name' in missing_options and - 'tenant_name' in specified_options): - LOG.warning("tenant_name is deprecated, use project_name " - "instead") - missing_options.remove('project_name') - if missing_options: - missing_options = ', '.join(missing_options) - raise exception.MissingRequiredConfigOptions( - missing_options=missing_options) - return loaded_driver + try: + loaded_driver = self.get_driver_info(driver) + except exception.DriverNotFound: + raise exception.InvalidDriver(driver=req) - # If we get here no datasource driver match was found. - raise exception.InvalidDriver(driver=req) + specified_options = set(config.keys()) + valid_options = set(loaded_driver['config'].keys()) + # Check that all the specified options passed in are + # valid configuration options that the driver exposes. + invalid_options = specified_options - valid_options + if invalid_options: + raise exception.InvalidDriverOption( + invalid_options=invalid_options) + + # check that all the required options are passed in + required_options = set( + [k for k, v in loaded_driver['config'].items() + if v == constants.REQUIRED]) + missing_options = required_options - specified_options + + if ('project_name' in missing_options and 'tenant_name' in + specified_options): + LOG.warning("tenant_name is deprecated, use project_name instead") + missing_options.remove('project_name') + + if missing_options: + missing_options = ', '.join(missing_options) + raise exception.MissingRequiredConfigOptions( + missing_options=missing_options) + return loaded_driver # Note (thread-safety): blocking function def create_datasource_service(self, datasource): @@ -657,13 +660,9 @@ class DseNode(object): LOG.info("datasource %s not enabled, skip loading", ds_dict['name']) return - - driver_info = self.get_driver_info(ds_dict['driver']) - # split class_path into module and class name - class_path = driver_info['module'] - pieces = class_path.split(".") - module_name = ".".join(pieces[:-1]) - class_name = pieces[-1] + driver = self.loaded_drivers.get(ds_dict['driver']) + if not driver: + raise exception.DriverNotFound(id=ds_dict['driver']) if ds_dict['config'] is None: args = {'ds_id': ds_dict['id']} @@ -671,18 +670,14 @@ class DseNode(object): args = dict(ds_dict['config'], ds_id=ds_dict['id']) kwargs = {'name': ds_dict['name'], 'args': args} LOG.info("creating service %s with class %s and args %s", - ds_dict['name'], module_name, + ds_dict['name'], driver.plugin, strutils.mask_password(kwargs, "****")) - - # import the module try: - # Note(thread-safety): blocking call? - module = importutils.import_module(module_name) - service = getattr(module, class_name)(**kwargs) + service = driver.plugin(**kwargs) except Exception: msg = ("Error loading instance of module '%s'") - LOG.exception(msg, class_path) - raise exception.DataServiceError(msg % class_path) + LOG.exception(msg, driver.plugin) + raise exception.DataServiceError(msg % driver.plugin) return service diff --git a/congress/harness.py b/congress/harness.py index 569bd5467..b6dbf7ba2 100644 --- a/congress/harness.py +++ b/congress/harness.py @@ -73,6 +73,9 @@ def create2(node_id=None, bus_id=None, existing_node=None, # create services as required services = {} + # Load all configured drivers + dse_node.DseNode.load_drivers() + if datasources: LOG.info("Registering congress datasource services on node %s", node.node_id) diff --git a/congress/tests/api/test_driver_model.py b/congress/tests/api/test_driver_model.py index 4543bdfa0..d9501d8a1 100644 --- a/congress/tests/api/test_driver_model.py +++ b/congress/tests/api/test_driver_model.py @@ -20,6 +20,7 @@ from __future__ import absolute_import from congress.api import webservice from congress.tests.api import base as api_base from congress.tests import base +from congress.tests import helper class TestDriverModel(base.SqlTestCase): @@ -46,15 +47,11 @@ class TestDriverModel(base.SqlTestCase): def test_drivers_list(self): context = {} - expected_ret = {"results": [ - { - "description": "This is a fake driver used for testing", - "id": "fake_datasource" - } - ]} - - ret = self.driver_model.get_items({}, context) - self.assertEqual(expected_ret, ret) + drivers = helper.supported_drivers() + expected_ret = sorted(drivers, key=lambda d: d['id']) + ret = self.driver_model.get_items({}, context)['results'] + actual_ret = sorted(ret, key=lambda d: d['id']) + self.assertEqual(expected_ret, actual_ret) def test_driver_details(self): context = { @@ -75,7 +72,6 @@ class TestDriverModel(base.SqlTestCase): }, "description": "This is a fake driver used for testing", "id": "fake_datasource", - "module": "congress.tests.fake_datasource.FakeDataSource", "secret": ["password"], "tables": [{'columns': [ {'description': None, 'name': 'id'}, diff --git a/congress/tests/dse2/test_datasource.py b/congress/tests/dse2/test_datasource.py index 6da7d55a8..654ac82eb 100644 --- a/congress/tests/dse2/test_datasource.py +++ b/congress/tests/dse2/test_datasource.py @@ -18,7 +18,6 @@ from __future__ import division from __future__ import absolute_import import mock -from oslo_config import cfg from oslo_db import exception as db_exc from congress.db import datasources as datasource_db @@ -194,12 +193,3 @@ class TestDataSource(base.SqlTestCase): # self.assertEqual( # schema, # fake_datasource.FakeDataSource.get_schema()) - - def test_duplicate_driver_name_raises(self): - # Load the driver twice - cfg.CONF.set_override( - 'drivers', - ['congress.tests.fake_datasource.FakeDataSource', - 'congress.tests.fake_datasource.FakeDataSource']) - self.assertRaises(congressException.BadConfig, - self.dseNode.load_drivers) diff --git a/congress/tests/dse2/test_dse_node.py b/congress/tests/dse2/test_dse_node.py index 0c4be989c..5efd43096 100644 --- a/congress/tests/dse2/test_dse_node.py +++ b/congress/tests/dse2/test_dse_node.py @@ -315,14 +315,14 @@ class TestDseNode(base.SqlTestCase): 'password': '', 'tenant_name': 'armax'}} + @mock.patch.object(dse_node.DseNode, 'validate_create_datasource') @mock.patch.object(dse_node.DseNode, 'get_driver_info') - def test_missing_driver_datasources(self, mock_driver_info): + def test_missing_driver_datasources(self, mock_driver_info, mock_validate): services = api_base.setup_config(api=False, policy=False) node = services['node'] ds_manager = services['ds_manager'] ds = self._get_datasource_request() - mock_driver_info.return_value = {'secret': [], - 'module': mock.MagicMock()} + mock_driver_info.return_value = {'secret': []} ds_manager.add_datasource(ds) mock_driver_info.side_effect = [exception.DriverNotFound] node.delete_missing_driver_datasources() diff --git a/congress/tests/helper.py b/congress/tests/helper.py index 2ba5e6e25..a2e67317b 100644 --- a/congress/tests/helper.py +++ b/congress/tests/helper.py @@ -458,3 +458,59 @@ class TestFailureException(Exception): """ def __init__(self, *args, **kwargs): Exception.__init__(self, *args, **kwargs) + + +def supported_drivers(): + """Get list of supported drivers by congress""" + + results = [ + {"id": "monasca", + "description": "Datasource driver that interfaces with monasca."}, + {"id": "plexxi", + "description": "Datasource driver that interfaces with PlexxiCore."}, + {"id": "doctor", + "description": "Datasource driver that allows external systems " + "to push data in accordance with OPNFV Doctor " + "Inspector southbound interface specification."}, + {"id": "aodh", + "description": "Datasource driver that interfaces with aodh."}, + {"id": "neutronv2_qos", + "description": "Datasource driver that interfaces with QoS " + "extension of OpenStack Networking aka Neutron."}, + {"id": "cloudfoundryv2", + "description": "Datasource driver that interfaces with cloudfoundry"}, + {"id": "heat", + "description": "Datasource driver that interfaces with OpenStack " + "orchestration aka heat."}, + {"id": "nova", + "description": "Datasource driver that interfaces with OpenStack " + "Compute aka nova."}, + {"id": "murano", + "description": "Datasource driver that interfaces with murano"}, + {"id": "neutronv2", + "description": "Datasource driver that interfaces with OpenStack " + "Networking aka Neutron."}, + {"id": "swift", + "description": "Datasource driver that interfaces with swift."}, + {"id": "ironic", + "description": "Datasource driver that interfaces with OpenStack " + "bare metal aka ironic."}, + {"id": "cinder", + "description": "Datasource driver that interfaces with OpenStack " + "cinder."}, + {"id": "fake_datasource", + "description": "This is a fake driver used for testing"}, + {"id": "config", + "description": "Datasource driver that allows OS configs retrieval."}, + {"id": "glancev2", + "description": "Datasource driver that interfaces with OpenStack " + "Images aka Glance."}, + {"id": "vcenter", + "description": "Datasource driver that interfaces with vcenter"}, + {"id": "keystonev3", + "description": "Datasource driver that interfaces with keystone."}, + {"id": "keystone", + "description": "Datasource driver that interfaces with keystone."}, + {"id": "mistral", + "description": "Datasource driver that interfaces with Mistral."}] + return results diff --git a/setup.cfg b/setup.cfg index 540461dcb..478168a31 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,6 +59,28 @@ console_scripts = congress-db-manage = congress.db.migration.cli:main congress-cfg-validator-agt = congress.cfg_validator.agent.agent:main +congress.datasource.drivers = + aodh = congress.datasources.aodh_driver:AodhDriver + cinder = congress.datasources.cinder_driver:CinderDriver + cloudfoundryv2 = congress.datasources.cloudfoundryv2_driver:CloudFoundryV2Driver + config = congress.datasources.cfgvalidator_driver:ValidatorDriver + doctor = congress.datasources.doctor_driver:DoctorDriver + fake_datasource = congress.tests.fake_datasource:FakeDataSource + glancev2 = congress.datasources.glancev2_driver:GlanceV2Driver + heat = congress.datasources.heatv1_driver:HeatV1Driver + ironic = congress.datasources.ironic_driver:IronicDriver + keystone = congress.datasources.keystone_driver:KeystoneDriver + keystonev3 = congress.datasources.keystonev3_driver:KeystoneV3Driver + mistral = congress.datasources.mistral_driver:MistralDriver + monasca = congress.datasources.monasca_driver:MonascaDriver + murano = congress.datasources.murano_driver:MuranoDriver + neutronv2 = congress.datasources.neutronv2_driver:NeutronV2Driver + neutronv2_qos = congress.datasources.neutronv2_qos_driver:NeutronV2QosDriver + nova = congress.datasources.nova_driver:NovaDriver + plexxi = congress.datasources.plexxi_driver:PlexxiDriver + swift = congress.datasources.swift_driver:SwiftDriver + vcenter = congress.datasources.vCenter_driver:VCenterDriver + [build_sphinx] all_files = 1 build-dir = doc/build