openstack
/
freezer-api
Code Issues Proposed changes

config options for elasticsearch connection 

Add support for options to be passed the the elasticsearch connection

The following options are supported in the freezer-api.conf file:

  hosts='http://localhost:9200'
  use_ssl=False
  ca_certs=''
  use_ssl=False
  timeout=60
  retries=20

The provided values are the respective defaults.

Implements blueprint: api-conf-db-options

Change-Id: I373503bf845d0ac8091c83280ac21611d30d1a07
This commit is contained in:
Fabrizio Vanni 2015-09-04 15:31:18 +01:00 committed by Pierre-Arthur MATHIEU
parent 7eb2c5b666
commit 4deb7ead04
5 changed files with 106 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
etc/freezer-api.conf
View File

@ -33,4 +33,10 @@ delay_auth_decision = False
[storage]
db=elasticsearch
endpoint=http://elasticsearch_host:9200
hosts='http://elasticsearch_host:9200'
#use_ssl=False
#ca_certs=''
#use_ssl=False
#timeout=60
#retries=20

52
freezer_api/storage/driver.py
View File

@ -20,6 +20,7 @@ Hudson (tjh@cryptsoft.com).
"""
import logging
import os
from oslo_config import cfg
@ -33,10 +34,32 @@ opt_group = cfg.OptGroup(name='storage',
storage_opts = [
cfg.StrOpt('db',
default='elasticsearch',
help='specify the storage db to use: elasticsearch (default)'),
help='specify the storage db to use (default: elasticsearch'),
# use of 'endpoint' parameter name is deprecated, please use 'hosts'
cfg.StrOpt('endpoint',
default='',
help='specify the storage hosts (deprecated, use "hosts"'),
cfg.StrOpt('hosts',
default='http://localhost:9200',
help='specify the storage endpoint')
help='specify the storage hosts'),
cfg.StrOpt('index',
default='freezer',
help='specify the name of the elasticsearch index'),
cfg.IntOpt('timeout',
default=60,
help='specify the connection timeout'),
cfg.IntOpt('retries',
default=20,
help='number of retries to allow before raising and error'),
cfg.BoolOpt('use_ssl',
default=False,
help='explicitly turn on SSL'),
cfg.BoolOpt('verify_certs',
default=False,
help='turn on SSL certs verification'),
cfg.StrOpt('ca_certs',
default=None,
help='path to CA certs on disk'),
]
CONF = cfg.CONF
@ -44,12 +67,29 @@ CONF.register_group(opt_group)
CONF.register_opts(storage_opts, opt_group)
def get_options():
if CONF.storage.ca_certs:
if not os.path.isfile(CONF.storage.ca_certs):
raise Exception('Elasticsearch configuration error: '
'CA_certs file not found ({0})'
.format(CONF.storage.ca_certs))
hosts = CONF.storage.endpoint or CONF.storage.hosts
if not hosts:
raise Exception('Elasticsearch configuration error: no host specified')
opts = dict(CONF.storage)
opts.pop('endpoint')
opts['hosts'] = hosts.split(',')
return opts
def get_db():
db_engine = CONF.storage.db
opts = get_options()
db_engine = opts.pop('db')
if db_engine == 'elasticsearch':
endpoint = CONF.storage.endpoint
logging.info(_LI('Storage backend: Elasticsearch at %s') % endpoint)
db = elastic.ElasticSearchEngine(endpoint)
logging.debug(_LI('Elastichsearch config options: %s') % str(opts))
db = elastic.ElasticSearchEngine(**opts)
else:
raise Exception(_('Database Engine %s not supported') % db_engine)
return db

7
freezer_api/storage/elastic.py
View File

@ -262,10 +262,11 @@ class SessionTypeManager(TypeManager):
class ElasticSearchEngine(object):
def __init__(self, hosts, index='freezer'):
def __init__(self, index='freezer', **kwargs):
self.index = index
self.es = elasticsearch.Elasticsearch(hosts)
logging.info(_LI('Using Elasticsearch host %s') % hosts)
self.es = elasticsearch.Elasticsearch(**kwargs)
logging.info(_LI('Storage backend: Elasticsearch '
'at %s') % kwargs['hosts'])
self.backup_manager = BackupTypeManager(self.es, 'backups')
self.client_manager = ClientTypeManager(self.es, 'clients')
self.job_manager = JobTypeManager(self.es, 'jobs')

50
tests/test_driver.py
View File

@ -22,23 +22,51 @@ Hudson (tjh@cryptsoft.com).
import unittest
from mock import patch
from mock import Mock, patch
from oslo_config import cfg
from freezer_api.common.exceptions import *
from freezer_api.storage import driver, elastic
from freezer_api.storage import driver
class TestStorageDriver(unittest.TestCase):
@patch('freezer_api.storage.elastic.logging')
@patch('freezer_api.storage.driver.logging')
def test_get_db_raises_when_db_not_supported(self, mock_logging):
cfg.CONF.storage.db = 'nodb'
mock_CONF = Mock()
mock_CONF.storage.db = 'nodb'
driver.CONF = mock_CONF
self.assertRaises(Exception, driver.get_db)
@patch('freezer_api.storage.elastic.logging')
def test_get_db_elastic(self, mock_logging):
cfg.CONF.storage.db = 'elasticsearch'
@patch('freezer_api.storage.driver.elastic')
@patch('freezer_api.storage.driver.logging')
def test_get_db_elastic(self, mock_logging, mock_elastic):
db = driver.get_db()
self.assertIsInstance(db, elastic.ElasticSearchEngine)
self.assertTrue(mock_elastic.ElasticSearchEngine)
@patch('freezer_api.storage.driver.elastic')
@patch('freezer_api.storage.driver.logging')
def test_get_db_elastic_raises_Exception_when_cert_file_not_found(self, mock_logging, mock_elastic):
mock_CONF = Mock()
mock_CONF.storage.db = 'elasticsearch'
mock_CONF.storage.hosts = 'es_server'
mock_CONF.storage.verify_certs = 'False'
mock_CONF.storage.ca_certs = 'not_existant'
mock_CONF.storage.use_ssl = False
mock_CONF.storage.timeout = 43
mock_CONF.storage.retries = 37
driver.CONF = mock_CONF
self.assertRaises(Exception, driver.get_db)
@patch('freezer_api.storage.driver.elastic')
@patch('freezer_api.storage.driver.logging')
def test_get_db_elastic_raises_Exception_when_hosts_not_defined(self, mock_logging, mock_elastic):
mock_CONF = Mock()
mock_CONF.storage.db = 'elasticsearch'
mock_CONF.storage.hosts = ''
mock_CONF.storage.endpoint = ''
mock_CONF.storage.verify_certs = 'False'
mock_CONF.storage.ca_certs = ''
mock_CONF.storage.use_ssl = False
mock_CONF.storage.timeout = 43
mock_CONF.storage.retries = 37
driver.CONF = mock_CONF
self.assertRaises(Exception, driver.get_db)

15
tests/test_elastic.py
View File

@ -376,7 +376,8 @@ class TestElasticSearchEngine_backup(unittest.TestCase):
@patch('freezer_api.storage.elastic.elasticsearch')
def setUp(self, mock_logging, mock_elasticsearch):
mock_elasticsearch.Elasticsearch.return_value = Mock()
self.eng = elastic.ElasticSearchEngine('http://elasticservaddr:1997')
kwargs = {'hosts': 'http://elasticservaddr:1997'}
self.eng = elastic.ElasticSearchEngine(index='freezer', **kwargs)
self.eng.backup_manager = Mock()
def test_get_backup_userid_and_backup_id_return_ok(self):
@ -486,7 +487,8 @@ class TestElasticSearchEngine_client(unittest.TestCase):
@patch('freezer_api.storage.elastic.elasticsearch')
def setUp(self, mock_logging, mock_elasticsearch):
mock_elasticsearch.Elasticsearch.return_value = Mock()
self.eng = elastic.ElasticSearchEngine('http://elasticservaddr:1997')
kwargs = {'hosts': 'http://elasticservaddr:1997'}
self.eng = elastic.ElasticSearchEngine(index='freezer', **kwargs)
self.eng.client_manager = Mock()
def test_get_client_userid_and_client_id_return_1elem_list_(self):
@ -579,7 +581,8 @@ class TestElasticSearchEngine_job(unittest.TestCase):
@patch('freezer_api.storage.elastic.elasticsearch')
def setUp(self, mock_elasticsearch, mock_logging):
mock_elasticsearch.Elasticsearch.return_value = Mock()
self.eng = elastic.ElasticSearchEngine('http://elasticservaddr:1997')
kwargs = {'hosts': 'http://elasticservaddr:1997'}
self.eng = elastic.ElasticSearchEngine(index='freezer', **kwargs)
self.eng.job_manager = Mock()
def test_get_job_userid_and_job_id_return_doc(self):
@ -713,7 +716,8 @@ class TestElasticSearchEngine_action(unittest.TestCase):
@patch('freezer_api.storage.elastic.elasticsearch')
def setUp(self, mock_elasticsearch, mock_logging):
mock_elasticsearch.Elasticsearch.return_value = Mock()
self.eng = elastic.ElasticSearchEngine('http://elasticservaddr:1997')
kwargs = {'hosts': 'http://elasticservaddr:1997'}
self.eng = elastic.ElasticSearchEngine(index='freezer', **kwargs)
self.eng.action_manager = Mock()
def test_get_action_userid_and_action_id_return_doc(self):
@ -847,7 +851,8 @@ class TestElasticSearchEngine_session(unittest.TestCase):
@patch('freezer_api.storage.elastic.elasticsearch')
def setUp(self, mock_elasticsearch, mock_logging):
mock_elasticsearch.Elasticsearch.return_value = Mock()
self.eng = elastic.ElasticSearchEngine('http://elasticservaddr:1997')
kwargs = {'hosts': 'http://elasticservaddr:1997'}
self.eng = elastic.ElasticSearchEngine(index='freezer', **kwargs)
self.eng.session_manager = Mock()
def test_get_session_userid_and_session_id_return_doc(self):