diff --git a/etc/freezer-api.conf b/etc/freezer-api.conf index 114df61e..9eb8621f 100644 --- a/etc/freezer-api.conf +++ b/etc/freezer-api.conf @@ -33,4 +33,10 @@ delay_auth_decision = False [storage] db=elasticsearch -endpoint=http://elasticsearch_host:9200 +hosts='http://elasticsearch_host:9200' + +#use_ssl=False +#ca_certs='' +#use_ssl=False +#timeout=60 +#retries=20 diff --git a/freezer_api/storage/driver.py b/freezer_api/storage/driver.py index 7646cf56..c571a08f 100644 --- a/freezer_api/storage/driver.py +++ b/freezer_api/storage/driver.py @@ -20,6 +20,7 @@ Hudson (tjh@cryptsoft.com). """ import logging +import os from oslo_config import cfg @@ -33,10 +34,32 @@ opt_group = cfg.OptGroup(name='storage', storage_opts = [ cfg.StrOpt('db', default='elasticsearch', - help='specify the storage db to use: elasticsearch (default)'), + help='specify the storage db to use (default: elasticsearch'), + # use of 'endpoint' parameter name is deprecated, please use 'hosts' cfg.StrOpt('endpoint', + default='', + help='specify the storage hosts (deprecated, use "hosts"'), + cfg.StrOpt('hosts', default='http://localhost:9200', - help='specify the storage endpoint') + help='specify the storage hosts'), + cfg.StrOpt('index', + default='freezer', + help='specify the name of the elasticsearch index'), + cfg.IntOpt('timeout', + default=60, + help='specify the connection timeout'), + cfg.IntOpt('retries', + default=20, + help='number of retries to allow before raising and error'), + cfg.BoolOpt('use_ssl', + default=False, + help='explicitly turn on SSL'), + cfg.BoolOpt('verify_certs', + default=False, + help='turn on SSL certs verification'), + cfg.StrOpt('ca_certs', + default=None, + help='path to CA certs on disk'), ] CONF = cfg.CONF @@ -44,12 +67,29 @@ CONF.register_group(opt_group) CONF.register_opts(storage_opts, opt_group) +def get_options(): + if CONF.storage.ca_certs: + if not os.path.isfile(CONF.storage.ca_certs): + raise Exception('Elasticsearch configuration error: ' + 'CA_certs file not found ({0})' + .format(CONF.storage.ca_certs)) + + hosts = CONF.storage.endpoint or CONF.storage.hosts + if not hosts: + raise Exception('Elasticsearch configuration error: no host specified') + + opts = dict(CONF.storage) + opts.pop('endpoint') + opts['hosts'] = hosts.split(',') + return opts + + def get_db(): - db_engine = CONF.storage.db + opts = get_options() + db_engine = opts.pop('db') if db_engine == 'elasticsearch': - endpoint = CONF.storage.endpoint - logging.info(_LI('Storage backend: Elasticsearch at %s') % endpoint) - db = elastic.ElasticSearchEngine(endpoint) + logging.debug(_LI('Elastichsearch config options: %s') % str(opts)) + db = elastic.ElasticSearchEngine(**opts) else: raise Exception(_('Database Engine %s not supported') % db_engine) return db diff --git a/freezer_api/storage/elastic.py b/freezer_api/storage/elastic.py index c6b2bc11..95874718 100644 --- a/freezer_api/storage/elastic.py +++ b/freezer_api/storage/elastic.py @@ -262,10 +262,11 @@ class SessionTypeManager(TypeManager): class ElasticSearchEngine(object): - def __init__(self, hosts, index='freezer'): + def __init__(self, index='freezer', **kwargs): self.index = index - self.es = elasticsearch.Elasticsearch(hosts) - logging.info(_LI('Using Elasticsearch host %s') % hosts) + self.es = elasticsearch.Elasticsearch(**kwargs) + logging.info(_LI('Storage backend: Elasticsearch ' + 'at %s') % kwargs['hosts']) self.backup_manager = BackupTypeManager(self.es, 'backups') self.client_manager = ClientTypeManager(self.es, 'clients') self.job_manager = JobTypeManager(self.es, 'jobs') diff --git a/tests/test_driver.py b/tests/test_driver.py index f4169ce4..9f7a43e8 100644 --- a/tests/test_driver.py +++ b/tests/test_driver.py @@ -22,23 +22,51 @@ Hudson (tjh@cryptsoft.com). import unittest -from mock import patch +from mock import Mock, patch -from oslo_config import cfg - -from freezer_api.common.exceptions import * -from freezer_api.storage import driver, elastic +from freezer_api.storage import driver class TestStorageDriver(unittest.TestCase): - @patch('freezer_api.storage.elastic.logging') + @patch('freezer_api.storage.driver.logging') def test_get_db_raises_when_db_not_supported(self, mock_logging): - cfg.CONF.storage.db = 'nodb' + mock_CONF = Mock() + mock_CONF.storage.db = 'nodb' + driver.CONF = mock_CONF self.assertRaises(Exception, driver.get_db) - @patch('freezer_api.storage.elastic.logging') - def test_get_db_elastic(self, mock_logging): - cfg.CONF.storage.db = 'elasticsearch' + @patch('freezer_api.storage.driver.elastic') + @patch('freezer_api.storage.driver.logging') + def test_get_db_elastic(self, mock_logging, mock_elastic): db = driver.get_db() - self.assertIsInstance(db, elastic.ElasticSearchEngine) + self.assertTrue(mock_elastic.ElasticSearchEngine) + + @patch('freezer_api.storage.driver.elastic') + @patch('freezer_api.storage.driver.logging') + def test_get_db_elastic_raises_Exception_when_cert_file_not_found(self, mock_logging, mock_elastic): + mock_CONF = Mock() + mock_CONF.storage.db = 'elasticsearch' + mock_CONF.storage.hosts = 'es_server' + mock_CONF.storage.verify_certs = 'False' + mock_CONF.storage.ca_certs = 'not_existant' + mock_CONF.storage.use_ssl = False + mock_CONF.storage.timeout = 43 + mock_CONF.storage.retries = 37 + driver.CONF = mock_CONF + self.assertRaises(Exception, driver.get_db) + + @patch('freezer_api.storage.driver.elastic') + @patch('freezer_api.storage.driver.logging') + def test_get_db_elastic_raises_Exception_when_hosts_not_defined(self, mock_logging, mock_elastic): + mock_CONF = Mock() + mock_CONF.storage.db = 'elasticsearch' + mock_CONF.storage.hosts = '' + mock_CONF.storage.endpoint = '' + mock_CONF.storage.verify_certs = 'False' + mock_CONF.storage.ca_certs = '' + mock_CONF.storage.use_ssl = False + mock_CONF.storage.timeout = 43 + mock_CONF.storage.retries = 37 + driver.CONF = mock_CONF + self.assertRaises(Exception, driver.get_db) diff --git a/tests/test_elastic.py b/tests/test_elastic.py index 2456c747..f9c74154 100644 --- a/tests/test_elastic.py +++ b/tests/test_elastic.py @@ -376,7 +376,8 @@ class TestElasticSearchEngine_backup(unittest.TestCase): @patch('freezer_api.storage.elastic.elasticsearch') def setUp(self, mock_logging, mock_elasticsearch): mock_elasticsearch.Elasticsearch.return_value = Mock() - self.eng = elastic.ElasticSearchEngine('http://elasticservaddr:1997') + kwargs = {'hosts': 'http://elasticservaddr:1997'} + self.eng = elastic.ElasticSearchEngine(index='freezer', **kwargs) self.eng.backup_manager = Mock() def test_get_backup_userid_and_backup_id_return_ok(self): @@ -486,7 +487,8 @@ class TestElasticSearchEngine_client(unittest.TestCase): @patch('freezer_api.storage.elastic.elasticsearch') def setUp(self, mock_logging, mock_elasticsearch): mock_elasticsearch.Elasticsearch.return_value = Mock() - self.eng = elastic.ElasticSearchEngine('http://elasticservaddr:1997') + kwargs = {'hosts': 'http://elasticservaddr:1997'} + self.eng = elastic.ElasticSearchEngine(index='freezer', **kwargs) self.eng.client_manager = Mock() def test_get_client_userid_and_client_id_return_1elem_list_(self): @@ -579,7 +581,8 @@ class TestElasticSearchEngine_job(unittest.TestCase): @patch('freezer_api.storage.elastic.elasticsearch') def setUp(self, mock_elasticsearch, mock_logging): mock_elasticsearch.Elasticsearch.return_value = Mock() - self.eng = elastic.ElasticSearchEngine('http://elasticservaddr:1997') + kwargs = {'hosts': 'http://elasticservaddr:1997'} + self.eng = elastic.ElasticSearchEngine(index='freezer', **kwargs) self.eng.job_manager = Mock() def test_get_job_userid_and_job_id_return_doc(self): @@ -713,7 +716,8 @@ class TestElasticSearchEngine_action(unittest.TestCase): @patch('freezer_api.storage.elastic.elasticsearch') def setUp(self, mock_elasticsearch, mock_logging): mock_elasticsearch.Elasticsearch.return_value = Mock() - self.eng = elastic.ElasticSearchEngine('http://elasticservaddr:1997') + kwargs = {'hosts': 'http://elasticservaddr:1997'} + self.eng = elastic.ElasticSearchEngine(index='freezer', **kwargs) self.eng.action_manager = Mock() def test_get_action_userid_and_action_id_return_doc(self): @@ -847,7 +851,8 @@ class TestElasticSearchEngine_session(unittest.TestCase): @patch('freezer_api.storage.elastic.elasticsearch') def setUp(self, mock_elasticsearch, mock_logging): mock_elasticsearch.Elasticsearch.return_value = Mock() - self.eng = elastic.ElasticSearchEngine('http://elasticservaddr:1997') + kwargs = {'hosts': 'http://elasticservaddr:1997'} + self.eng = elastic.ElasticSearchEngine(index='freezer', **kwargs) self.eng.session_manager = Mock() def test_get_session_userid_and_session_id_return_doc(self):