From 82a708abcf72892c0864c9beef1c412cf3b23194 Mon Sep 17 00:00:00 2001 From: Anusha Ramineni Date: Tue, 16 Jan 2018 13:01:31 +0530 Subject: [PATCH] Loads all available drivers by default This commit loads all datasources drivers by default without the need of config option. The config option 'drivers' would be deprecated as it is required to update manually on newly supported drivers for each release and also error prone. To disable any of the drivers, can be disabled using the config option 'disabled_drivers' For example, disabled_drivers = plexxi, murano TODO: Support disabled_drivers option to disable the datasource drivers Partially-Implements blueprint enable-drivers-by-default Change-Id: I5f9878d07c2e1487f8f14f7cd1948560327d8083 --- congress/api/system/driver_model.py | 2 +- congress/common/config.py | 2 + congress/dse2/dse_node.py | 131 ++++++++++++------------ congress/harness.py | 3 + congress/tests/api/test_driver_model.py | 16 ++- congress/tests/dse2/test_datasource.py | 10 -- congress/tests/dse2/test_dse_node.py | 6 +- congress/tests/helper.py | 56 ++++++++++ setup.cfg | 22 ++++ 9 files changed, 156 insertions(+), 92 deletions(-) diff --git a/congress/api/system/driver_model.py b/congress/api/system/driver_model.py index 0161419a4..27ec589eb 100644 --- a/congress/api/system/driver_model.py +++ b/congress/api/system/driver_model.py @@ -40,7 +40,7 @@ class DatasourceDriverModel(base.APIModel): drivers = self.bus.get_drivers_info() fields = ['id', 'description'] results = [self.bus.make_datasource_dict( - drivers[driver], fields=fields) + driver, fields=fields) for driver in drivers] return {"results": results} diff --git a/congress/common/config.py b/congress/common/config.py index 72adae438..5bf62fceb 100644 --- a/congress/common/config.py +++ b/congress/common/config.py @@ -52,6 +52,8 @@ core_opts = [ help=_('The type of authentication to use')), cfg.ListOpt('drivers', default=[], + deprecated_for_removal=True, + deprecated_reason='automatically loads all configured drivers', help=_('List of driver class paths to import.')), cfg.IntOpt('datasource_sync_period', default=60, help='The number of seconds to wait between synchronizing ' diff --git a/congress/dse2/dse_node.py b/congress/dse2/dse_node.py index af9636715..db2bc9c53 100644 --- a/congress/dse2/dse_node.py +++ b/congress/dse2/dse_node.py @@ -22,9 +22,9 @@ from oslo_log import log as logging import oslo_messaging as messaging from oslo_messaging import exceptions as messaging_exceptions from oslo_messaging.rpc import dispatcher -from oslo_utils import importutils from oslo_utils import strutils from oslo_utils import uuidutils +import stevedore from congress.datalog import compile as datalog_compile from congress.datasources import constants @@ -56,6 +56,8 @@ class DseNode(object): CONTROL_TOPIC = 'congress-control' SERVICE_TOPIC_PREFIX = 'congress-service-' + loaded_drivers = {} + def node_rpc_target(self, namespace=None, server=None, fanout=False): return messaging.Target(exchange=self.EXCHANGE, topic=self._add_partition(self.CONTROL_TOPIC), @@ -119,7 +121,7 @@ class DseNode(object): self._control_bus = control_bus.DseNodeControlBus(self) self.register_service(self._control_bus) # load configured drivers - self.loaded_drivers = self.load_drivers() + # self.loaded_drivers = self.load_drivers() self.periodic_tasks = None self.sync_thread = None self.start() @@ -513,36 +515,35 @@ class DseNode(object): s._published_tables_with_subscriber = tables_with_subs # Driver CRUD. Maybe belongs in a subclass of DseNode? - # Note(thread-safety): blocking function? - def load_drivers(self): - """Load all configured drivers and check no name conflict""" + @classmethod + def load_drivers(cls): + """Loads all configured drivers""" result = {} - for driver_path in cfg.CONF.drivers: - # Note(thread-safety): blocking call? - obj = importutils.import_class(driver_path) - driver = obj.get_datasource_info() - if driver['id'] in result: - raise exception.BadConfig(_("There is a driver loaded already" - "with the driver name of %s") - % driver['id']) - driver['module'] = driver_path - result[driver['id']] = driver - return result + mgr = stevedore.extension.ExtensionManager( + namespace='congress.datasource.drivers', + invoke_on_load=False) - def get_driver_info(self, driver_name): - driver = self.loaded_drivers.get(driver_name) + for driver in mgr: + result[driver.name] = driver + + cls.loaded_drivers = result + + @classmethod + def get_driver_info(cls, driver_name): + driver = cls.loaded_drivers.get(driver_name) if not driver: raise exception.DriverNotFound(id=driver_name) - return driver + return driver.plugin.get_datasource_info() - def get_drivers_info(self): - return self.loaded_drivers + @classmethod + def get_drivers_info(cls): + drivers = cls.loaded_drivers.values() + return [d.plugin.get_datasource_info() for d in drivers] - def get_driver_schema(self, drivername): - driver = self.get_driver_info(drivername) - # Note(thread-safety): blocking call? - obj = importutils.import_class(driver['module']) - return obj.get_schema() + @classmethod + def get_driver_schema(cls, drivername): + driver = cls.loaded_drivers.get(drivername) + return driver.plugin.get_schema() # Datasource CRUD. Maybe belongs in a subclass of DseNode? # Note(thread-safety): blocking function @@ -618,35 +619,37 @@ class DseNode(object): raise exception.InvalidDatasourceName(value=name) driver = req['driver'] config = req['config'] or {} - for loaded_driver in self.loaded_drivers.values(): - if loaded_driver['id'] == driver: - specified_options = set(config.keys()) - valid_options = set(loaded_driver['config'].keys()) - # Check that all the specified options passed in are - # valid configuration options that the driver exposes. - invalid_options = specified_options - valid_options - if invalid_options: - raise exception.InvalidDriverOption( - invalid_options=invalid_options) - # check that all the required options are passed in - required_options = set( - [k for k, v in loaded_driver['config'].items() - if v == constants.REQUIRED]) - missing_options = required_options - specified_options - if ('project_name' in missing_options and - 'tenant_name' in specified_options): - LOG.warning("tenant_name is deprecated, use project_name " - "instead") - missing_options.remove('project_name') - if missing_options: - missing_options = ', '.join(missing_options) - raise exception.MissingRequiredConfigOptions( - missing_options=missing_options) - return loaded_driver + try: + loaded_driver = self.get_driver_info(driver) + except exception.DriverNotFound: + raise exception.InvalidDriver(driver=req) - # If we get here no datasource driver match was found. - raise exception.InvalidDriver(driver=req) + specified_options = set(config.keys()) + valid_options = set(loaded_driver['config'].keys()) + # Check that all the specified options passed in are + # valid configuration options that the driver exposes. + invalid_options = specified_options - valid_options + if invalid_options: + raise exception.InvalidDriverOption( + invalid_options=invalid_options) + + # check that all the required options are passed in + required_options = set( + [k for k, v in loaded_driver['config'].items() + if v == constants.REQUIRED]) + missing_options = required_options - specified_options + + if ('project_name' in missing_options and 'tenant_name' in + specified_options): + LOG.warning("tenant_name is deprecated, use project_name instead") + missing_options.remove('project_name') + + if missing_options: + missing_options = ', '.join(missing_options) + raise exception.MissingRequiredConfigOptions( + missing_options=missing_options) + return loaded_driver # Note (thread-safety): blocking function def create_datasource_service(self, datasource): @@ -667,13 +670,9 @@ class DseNode(object): LOG.info("datasource %s not enabled, skip loading", ds_dict['name']) return - - driver_info = self.get_driver_info(ds_dict['driver']) - # split class_path into module and class name - class_path = driver_info['module'] - pieces = class_path.split(".") - module_name = ".".join(pieces[:-1]) - class_name = pieces[-1] + driver = self.loaded_drivers.get(ds_dict['driver']) + if not driver: + raise exception.DriverNotFound(id=ds_dict['driver']) if ds_dict['config'] is None: args = {'ds_id': ds_dict['id']} @@ -681,18 +680,14 @@ class DseNode(object): args = dict(ds_dict['config'], ds_id=ds_dict['id']) kwargs = {'name': ds_dict['name'], 'args': args} LOG.info("creating service %s with class %s and args %s", - ds_dict['name'], module_name, + ds_dict['name'], driver.plugin, strutils.mask_password(kwargs, "****")) - - # import the module try: - # Note(thread-safety): blocking call? - module = importutils.import_module(module_name) - service = getattr(module, class_name)(**kwargs) + service = driver.plugin(**kwargs) except Exception: msg = ("Error loading instance of module '%s'") - LOG.exception(msg, class_path) - raise exception.DataServiceError(msg % class_path) + LOG.exception(msg, driver.plugin) + raise exception.DataServiceError(msg % driver.plugin) return service diff --git a/congress/harness.py b/congress/harness.py index 569bd5467..b6dbf7ba2 100644 --- a/congress/harness.py +++ b/congress/harness.py @@ -73,6 +73,9 @@ def create2(node_id=None, bus_id=None, existing_node=None, # create services as required services = {} + # Load all configured drivers + dse_node.DseNode.load_drivers() + if datasources: LOG.info("Registering congress datasource services on node %s", node.node_id) diff --git a/congress/tests/api/test_driver_model.py b/congress/tests/api/test_driver_model.py index 4543bdfa0..d9501d8a1 100644 --- a/congress/tests/api/test_driver_model.py +++ b/congress/tests/api/test_driver_model.py @@ -20,6 +20,7 @@ from __future__ import absolute_import from congress.api import webservice from congress.tests.api import base as api_base from congress.tests import base +from congress.tests import helper class TestDriverModel(base.SqlTestCase): @@ -46,15 +47,11 @@ class TestDriverModel(base.SqlTestCase): def test_drivers_list(self): context = {} - expected_ret = {"results": [ - { - "description": "This is a fake driver used for testing", - "id": "fake_datasource" - } - ]} - - ret = self.driver_model.get_items({}, context) - self.assertEqual(expected_ret, ret) + drivers = helper.supported_drivers() + expected_ret = sorted(drivers, key=lambda d: d['id']) + ret = self.driver_model.get_items({}, context)['results'] + actual_ret = sorted(ret, key=lambda d: d['id']) + self.assertEqual(expected_ret, actual_ret) def test_driver_details(self): context = { @@ -75,7 +72,6 @@ class TestDriverModel(base.SqlTestCase): }, "description": "This is a fake driver used for testing", "id": "fake_datasource", - "module": "congress.tests.fake_datasource.FakeDataSource", "secret": ["password"], "tables": [{'columns': [ {'description': None, 'name': 'id'}, diff --git a/congress/tests/dse2/test_datasource.py b/congress/tests/dse2/test_datasource.py index 6da7d55a8..654ac82eb 100644 --- a/congress/tests/dse2/test_datasource.py +++ b/congress/tests/dse2/test_datasource.py @@ -18,7 +18,6 @@ from __future__ import division from __future__ import absolute_import import mock -from oslo_config import cfg from oslo_db import exception as db_exc from congress.db import datasources as datasource_db @@ -194,12 +193,3 @@ class TestDataSource(base.SqlTestCase): # self.assertEqual( # schema, # fake_datasource.FakeDataSource.get_schema()) - - def test_duplicate_driver_name_raises(self): - # Load the driver twice - cfg.CONF.set_override( - 'drivers', - ['congress.tests.fake_datasource.FakeDataSource', - 'congress.tests.fake_datasource.FakeDataSource']) - self.assertRaises(congressException.BadConfig, - self.dseNode.load_drivers) diff --git a/congress/tests/dse2/test_dse_node.py b/congress/tests/dse2/test_dse_node.py index 0c4be989c..5efd43096 100644 --- a/congress/tests/dse2/test_dse_node.py +++ b/congress/tests/dse2/test_dse_node.py @@ -315,14 +315,14 @@ class TestDseNode(base.SqlTestCase): 'password': '', 'tenant_name': 'armax'}} + @mock.patch.object(dse_node.DseNode, 'validate_create_datasource') @mock.patch.object(dse_node.DseNode, 'get_driver_info') - def test_missing_driver_datasources(self, mock_driver_info): + def test_missing_driver_datasources(self, mock_driver_info, mock_validate): services = api_base.setup_config(api=False, policy=False) node = services['node'] ds_manager = services['ds_manager'] ds = self._get_datasource_request() - mock_driver_info.return_value = {'secret': [], - 'module': mock.MagicMock()} + mock_driver_info.return_value = {'secret': []} ds_manager.add_datasource(ds) mock_driver_info.side_effect = [exception.DriverNotFound] node.delete_missing_driver_datasources() diff --git a/congress/tests/helper.py b/congress/tests/helper.py index 2ba5e6e25..a2e67317b 100644 --- a/congress/tests/helper.py +++ b/congress/tests/helper.py @@ -458,3 +458,59 @@ class TestFailureException(Exception): """ def __init__(self, *args, **kwargs): Exception.__init__(self, *args, **kwargs) + + +def supported_drivers(): + """Get list of supported drivers by congress""" + + results = [ + {"id": "monasca", + "description": "Datasource driver that interfaces with monasca."}, + {"id": "plexxi", + "description": "Datasource driver that interfaces with PlexxiCore."}, + {"id": "doctor", + "description": "Datasource driver that allows external systems " + "to push data in accordance with OPNFV Doctor " + "Inspector southbound interface specification."}, + {"id": "aodh", + "description": "Datasource driver that interfaces with aodh."}, + {"id": "neutronv2_qos", + "description": "Datasource driver that interfaces with QoS " + "extension of OpenStack Networking aka Neutron."}, + {"id": "cloudfoundryv2", + "description": "Datasource driver that interfaces with cloudfoundry"}, + {"id": "heat", + "description": "Datasource driver that interfaces with OpenStack " + "orchestration aka heat."}, + {"id": "nova", + "description": "Datasource driver that interfaces with OpenStack " + "Compute aka nova."}, + {"id": "murano", + "description": "Datasource driver that interfaces with murano"}, + {"id": "neutronv2", + "description": "Datasource driver that interfaces with OpenStack " + "Networking aka Neutron."}, + {"id": "swift", + "description": "Datasource driver that interfaces with swift."}, + {"id": "ironic", + "description": "Datasource driver that interfaces with OpenStack " + "bare metal aka ironic."}, + {"id": "cinder", + "description": "Datasource driver that interfaces with OpenStack " + "cinder."}, + {"id": "fake_datasource", + "description": "This is a fake driver used for testing"}, + {"id": "config", + "description": "Datasource driver that allows OS configs retrieval."}, + {"id": "glancev2", + "description": "Datasource driver that interfaces with OpenStack " + "Images aka Glance."}, + {"id": "vcenter", + "description": "Datasource driver that interfaces with vcenter"}, + {"id": "keystonev3", + "description": "Datasource driver that interfaces with keystone."}, + {"id": "keystone", + "description": "Datasource driver that interfaces with keystone."}, + {"id": "mistral", + "description": "Datasource driver that interfaces with Mistral."}] + return results diff --git a/setup.cfg b/setup.cfg index 540461dcb..478168a31 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,6 +59,28 @@ console_scripts = congress-db-manage = congress.db.migration.cli:main congress-cfg-validator-agt = congress.cfg_validator.agent.agent:main +congress.datasource.drivers = + aodh = congress.datasources.aodh_driver:AodhDriver + cinder = congress.datasources.cinder_driver:CinderDriver + cloudfoundryv2 = congress.datasources.cloudfoundryv2_driver:CloudFoundryV2Driver + config = congress.datasources.cfgvalidator_driver:ValidatorDriver + doctor = congress.datasources.doctor_driver:DoctorDriver + fake_datasource = congress.tests.fake_datasource:FakeDataSource + glancev2 = congress.datasources.glancev2_driver:GlanceV2Driver + heat = congress.datasources.heatv1_driver:HeatV1Driver + ironic = congress.datasources.ironic_driver:IronicDriver + keystone = congress.datasources.keystone_driver:KeystoneDriver + keystonev3 = congress.datasources.keystonev3_driver:KeystoneV3Driver + mistral = congress.datasources.mistral_driver:MistralDriver + monasca = congress.datasources.monasca_driver:MonascaDriver + murano = congress.datasources.murano_driver:MuranoDriver + neutronv2 = congress.datasources.neutronv2_driver:NeutronV2Driver + neutronv2_qos = congress.datasources.neutronv2_qos_driver:NeutronV2QosDriver + nova = congress.datasources.nova_driver:NovaDriver + plexxi = congress.datasources.plexxi_driver:PlexxiDriver + swift = congress.datasources.swift_driver:SwiftDriver + vcenter = congress.datasources.vCenter_driver:VCenterDriver + [build_sphinx] all_files = 1 build-dir = doc/build