openstack
/
monasca-transform
Code Issues Proposed changes

Add TLS/SSL capability to database connection 

Add properties to conf file to allow configuration of
SSL for the database connection.  Done for both the python
and java connection strings.

Change-Id: I4c3d25c3f8f12eae801a6a818bf4ac7acd93d2dc
This commit is contained in:
darfed 2016-07-25 16:29:01 +01:00 committed by David C Kennedy
parent aa7addbc84
commit bb83b30dc1
10 changed files with 112 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
monasca_transform/config/config_initializer.py
View File

@ -60,7 +60,9 @@ class ConfigInitializer(object):
cfg.StrOpt('host'),
cfg.StrOpt('database_name'),
cfg.StrOpt('username'),
cfg.StrOpt('password')
cfg.StrOpt('password'),
cfg.BoolOpt('use_ssl', default=False),
cfg.StrOpt('ca_file')
]
mysql_group = cfg.OptGroup(name='database', title='database')
cfg.CONF.register_group(mysql_group)

23
monasca_transform/data_driven_specs/mysql_data_driven_specs_repo.py
View File

@ -13,12 +13,12 @@
# under the License.
import json
from oslo_config import cfg
from pyspark.sql import DataFrameReader
from monasca_transform.data_driven_specs.data_driven_specs_repo \
import DataDrivenSpecsRepo
from monasca_transform.db.db_utils import DbUtil
class MySQLDataDrivenSpecsRepo(DataDrivenSpecsRepo):
@ -26,23 +26,6 @@ class MySQLDataDrivenSpecsRepo(DataDrivenSpecsRepo):
transform_specs_data_frame = None
pre_transform_specs_data_frame = None
def __init__(self):
self.database_impl = cfg.CONF.database.server_type
self.database_name = cfg.CONF.database.database_name
self.database_server = cfg.CONF.database.host
self.database_uid = cfg.CONF.database.username
self.database_pwd = cfg.CONF.database.password
def get_connection_string(self):
# FIXME I don't like this, find a better way of managing the conn
return 'jdbc:%s://%s/%s?user=%s&password=%s' % (
self.database_impl,
self.database_server,
self.database_name,
self.database_uid,
self.database_pwd
)
def get_data_driven_specs(self, sql_context=None,
data_driven_spec_type=None):
data_driven_spec = None
@ -65,7 +48,7 @@ class MySQLDataDrivenSpecsRepo(DataDrivenSpecsRepo):
data_frame_reader = DataFrameReader(sql_context)
transform_specs_data_frame = data_frame_reader.jdbc(
self.get_connection_string(),
DbUtil.get_java_db_connection_string(),
'transform_specs'
)
data = []
@ -81,7 +64,7 @@ class MySQLDataDrivenSpecsRepo(DataDrivenSpecsRepo):
data_frame_reader = DataFrameReader(sql_context)
pre_transform_specs_data_frame = data_frame_reader.jdbc(
self.get_connection_string(),
DbUtil.get_java_db_connection_string(),
'pre_transform_specs'
)
data = []

0
monasca_transform/db/__init__.py Normal file
View File

41
monasca_transform/db/db_utils.py Normal file
View File

@ -0,0 +1,41 @@
from oslo_config import cfg
class DbUtil(object):
@staticmethod
def get_python_db_connection_string(config=cfg.CONF):
database_name = config.database.database_name
database_server = config.database.host
database_uid = config.database.username
database_pwd = config.database.password
if config.database.use_ssl:
db_ssl = "?ssl_ca=%s" % config.database.ca_file
else:
db_ssl = ''
return 'mysql+pymysql://%s:%s@%s/%s%s' % (
database_uid,
database_pwd,
database_server,
database_name,
db_ssl)
@staticmethod
def get_java_db_connection_string(config=cfg.CONF):
ssl_params = ''
if config.database.use_ssl:
ssl_params = "&useSSL=%s&requireSSL=%s" % (
config.database.use_ssl, config.database.use_ssl
)
# FIXME I don't like this, find a better way of managing the conn
return 'jdbc:%s://%s/%s?user=%s&password=%s%s' % (
config.database.server_type,
config.database.host,
config.database.database_name,
config.database.username,
config.database.password,
ssl_params,
)

12
monasca_transform/mysql_offset_specs.py
View File

@ -18,6 +18,7 @@ from sqlalchemy import desc
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import sessionmaker
from monasca_transform.db.db_utils import DbUtil
from monasca_transform.offset_specs import OffsetSpec
from monasca_transform.offset_specs import OffsetSpecs
@ -41,16 +42,9 @@ class MySQLOffsetSpec(Base, OffsetSpec):
class MySQLOffsetSpecs(OffsetSpecs):
def __init__(self):
database_name = cfg.CONF.database.database_name
database_server = cfg.CONF.database.host
database_uid = cfg.CONF.database.username
database_pwd = cfg.CONF.database.password
db = create_engine('mysql+pymysql://%s:%s@%s/%s' % (
database_uid,
database_pwd,
database_server,
database_name), isolation_level="READ UNCOMMITTED")
db = create_engine(DbUtil.get_python_db_connection_string(),
isolation_level="READ UNCOMMITTED")
db.echo = True
# reflect the tables

1
test-requirements.txt
View File

@ -4,3 +4,4 @@ flake8>=2.2.4
nose==1.3.0
mock>=1.0.1
tox
fixtures==1.4.0

5
tests/unit/config/config_initializer_test.py
View File

@ -40,6 +40,9 @@ class TestConfigInitializer(unittest.TestCase):
cfg.CONF.database.username)
self.assertEqual('test_database_password',
cfg.CONF.database.password)
self.assertEqual('test_ca_file_path',
cfg.CONF.database.ca_file)
self.assertTrue(cfg.CONF.database.use_ssl)
def test_use_default_config_file(self):
@ -64,3 +67,5 @@ class TestConfigInitializer(unittest.TestCase):
cfg.CONF.database.username)
self.assertEqual('password',
cfg.CONF.database.password)
self.assertIsNone(cfg.CONF.database.ca_file)
self.assertFalse(cfg.CONF.database.use_ssl)

0
tests/unit/db/__init__.py Normal file
View File

54
tests/unit/db/test_db_util.py Normal file
View File

@ -0,0 +1,54 @@
from oslo_config import cfg
from oslo_config.fixture import Config
import unittest
from monasca_transform.config.config_initializer import ConfigInitializer
from monasca_transform.db.db_utils import DbUtil
class TestDBUtil(unittest.TestCase):
def setUp(self):
ConfigInitializer.basic_config(
default_config_files=[
'tests/unit/test_resources/config/test_config.conf'
])
self.config = Config()
self.config.config(
group='database',
use_ssl=True,
host='test_ssl_hostname',
server_type='jdbc_driver',
database_name='db_name',
username='test_user',
password='pwd',
ca_file='ca_file')
self.config.setUp()
def tearDown(self):
self.config.cleanUp()
def test_get_java_db_connection_string_with_ssl(self):
self.assertEqual(
'jdbc:jdbc_driver://test_ssl_hostname/db_name?user=test_user'
'&password=pwd&useSSL=True&requireSSL=True',
DbUtil.get_java_db_connection_string(cfg.CONF))
def test_get_python_db_connection_string_with_ssl(self):
self.assertEqual(
'mysql+pymysql://test_user:pwd@test_ssl_hostname/db_name?'
'ssl_ca=ca_file',
DbUtil.get_python_db_connection_string(cfg.CONF))
def test_get_java_db_connection_string_without_ssl(self):
self.config.config(group='database', use_ssl=False)
self.assertEqual(
'jdbc:jdbc_driver://test_ssl_hostname/db_name?'
'user=test_user&password=pwd',
DbUtil.get_java_db_connection_string())
def test_get_python_db_connection_string_without_ssl(self):
self.config.config(group='database', use_ssl=False)
self.assertEqual(
'mysql+pymysql://test_user:pwd@test_ssl_hostname/db_name',
DbUtil.get_python_db_connection_string())

2
tests/unit/test_resources/config/test_config.conf
View File

@ -10,6 +10,8 @@ host = test_host_name
database_name = test_database_name
username = test_database_user_name
password = test_database_password
use_ssl = True
ca_file = test_ca_file_path
[stage_processors]
enable_pre_hourly_processor = False