Cleanup Ranger logging logic and authentication

Cleanup Ranger logging logic  and allow the following logging option:
- logging to console only
- logging to logfile only
= logging to console AND logfile

Also cleanup authentication logic as well.

Change-Id: I8e422a1c9e44c4f58078d93d4f2a5a009df91ba1
This commit is contained in:
stewie925 2018-09-20 13:29:38 -07:00 committed by Hari
parent 222fd98f16
commit d376b60597
14 changed files with 164 additions and 436 deletions

View File

@ -7,20 +7,24 @@ ssl_verify = False
ranger_agent_https_enabled = False
ranger_agent_client_cert_path = ''
log_location = '/var/log/ranger/'
# use_handlers determine where output logging is routed to
# valid values are 'console', 'logfile', or 'console,logfile'
use_handlers = 'console,logfile'
[keystone_authtoken]
username = 'admin'
password = 'password'
user_role= 'admin'
project_name = 'admin'
region = 'RegionOne'
project_domain_name = 'default'
user_domain_name = 'default'
## version may be '2.0' or '3' ('3' by default)
version = '2.0'
# Ranger shall be using keystone v3 by default
auth_version = 'v3'
auth_enabled = False
[database]
connection = 'mysql://user:pass@localhost:3306/'
connection = 'mysql://user:pass@localhost:3306/orm'
max_retries = 3
user_role = 'admin'
@ -50,8 +54,8 @@ log = 'rms.log'
[rds]
port = 8777
repo_local_location = '/home/stack/orm_repo'
repo_remote_location = 'git@127.0.0.1:/home/stack/orm_repo.git'
repo_local_location = '/opt/stack/ranger/orm_repo'
repo_remote_location = 'git@127.0.0.1:/opt/stack/ranger/orm_repo.git'
log = 'rdstest.log'
[cli]

View File

@ -14,8 +14,8 @@
# under the License.
from oslo_config import cfg
CONF = cfg.CONF
CONF = cfg.CONF
# Orm config options in DEFAULT block
OrmOpts = [
@ -45,7 +45,11 @@ OrmOpts = [
help='Orm log directory.'),
cfg.StrOpt('debug_level',
default='DEBUG',
help='logging debug level')
help='logging debug level'),
cfg.StrOpt('use_handlers',
choices=('console', 'logfile', 'console,logfile'),
default='console',
help='output logging to console or logfile or both'),
]
CONF.register_opts(OrmOpts)
@ -70,7 +74,7 @@ OrmAuthGroup = [
cfg.BoolOpt('auth_enabled',
default=False,
help='Auth token Enabled/Disabled.'),
cfg.StrOpt('version',
cfg.StrOpt('auth_version',
default='v3',
help='Keystone version number.'),
cfg.StrOpt('project_domain_name',
@ -80,6 +84,7 @@ OrmAuthGroup = [
default='default',
help='User domain name.'),
cfg.StrOpt('user_role',
default='admin',
help='token user role.')
]
@ -245,29 +250,26 @@ CONF.register_opts(OrmCliGroup, orm_cli_group)
# backward compatiblitiy as other modules still
# referenced to them.
################################################################
# logging config settings
debug_level = CONF.debug_level
handler_list = CONF.use_handlers.split(",")
# ranger settings
protocol = CONF.protocol
orm_host = CONF.orm_host
ranger_url = CONF.ranger_url
ranger_base = CONF.ranger_base
ranger_url = CONF.ranger_url
ssl_verify = CONF.ssl_verify
token_auth_enabled = CONF.keystone_authtoken.auth_enabled
token_auth_user = CONF.keystone_authtoken.username
token_auth_pass = CONF.keystone_authtoken.password
token_auth_tenant = CONF.keystone_authtoken.project_name
token_auth_user_role = CONF.keystone_authtoken.user_role
user_domain_name = CONF.keystone_authtoken.user_domain_name
project_domain_name = CONF.keystone_authtoken.project_domain_name
conn = CONF.database.connection
db_connect = conn.replace("mysql+pymysql", "mysql") if conn else None
# pass keystone version '2.0' or '3'
token_auth_version = '3'
ssl_verify = CONF.ssl_verify
token_auth_version = '3' if (CONF.keystone_authtoken.auth_version == 'v3') else '2.0'
cert_path = CONF.ranger_agent_client_cert_path
https_enabled = CONF.ranger_agent_https_enabled
uuid = {'port': CONF.uuid.port,
'base_url': '{}://{}:{}/'.
format(protocol, orm_host, CONF.uuid.port),
@ -312,3 +314,108 @@ rds = {'port': CONF.rds.port,
'log': '{}/{}'.format(CONF.log_location, CONF.rds.log)}
cli = {'base_region': CONF.cli.base_region}
def server_request_auth(server_name):
if server_name == 'cms':
policy_name = cms['policy_file']
elif server_name == 'fms':
policy_name = fms['policy_file']
elif server_name == 'ims':
policy_name = ims['policy_file']
elif server_name == 'rms':
policy_name = rms['policy_file']
else:
# policy_N/A for services 'rds', 'audit', and 'uuid'
policy_name = None
# authentication settings
request_authentication = {
"enabled": CONF.keystone_authtoken.auth_enabled,
# The Keystone version currently in use. For Ranger, use '3' by default.
"keystone_version": token_auth_version,
"mech_id": CONF.keystone_authtoken.username,
"mech_pass": CONF.keystone_authtoken.password,
"policy_file": policy_name,
# The Keystone collection under which the role was granted.
# The key can be either "tenant" (for Keystone v2.0) or "domain"
# (for Keystone v3) and the value is the tenant/domain name.
"role_location": {"domain": "admin"},
"tenant_name": CONF.keystone_authtoken.project_name,
"rms_url": rms['base_url'],
"project_domain_name": CONF.keystone_authtoken.project_domain_name,
"token_role": CONF.keystone_authtoken.user_role,
"user_domain_name": CONF.keystone_authtoken.user_domain_name
}
return request_authentication
def get_log_config(log_file_name, ranger_service, ranger_service_module):
# Ranger logging template - we want to have the option of not routing to logfiles
# for all loggers except 'pecan' and 'py.warnings', which only logs to console
logging_template = {
'root': {'level': 'INFO', 'handlers': handler_list},
'loggers': {
ranger_service_module: {
'level': debug_level,
'handlers': handler_list,
'propagate': False
},
'orm.common.orm_common': {
'level': debug_level,
'handlers': handler_list,
'propagate': False
},
'orm.common.client.keystone.keystone_utils': {
'level': debug_level,
'handlers': handler_list,
'propagate': False
},
'orm.common.client.audit.audit_client': {
'level': debug_level,
'handlers': handler_list,
'propagate': False
},
'pecan': {'level': debug_level, 'handlers': handler_list,
'propagate': False},
'py.warnings': {'handlers': handler_list},
'__force_dict__': True
},
'handlers': {
'console': {
'level': debug_level,
'class': 'logging.StreamHandler',
'formatter': 'color'
},
'logfile': {
'level': debug_level,
'class': 'logging.handlers.RotatingFileHandler',
'maxBytes': 50000000,
'backupCount': 10,
'filename': log_file_name,
'formatter': 'simple'
}
},
'formatters': {
'simple': {
'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]'
'[%(threadName)s] %(message)s')
},
'color': {
'()': 'pecan.log.ColorFormatter',
'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]'
'[%(threadName)s] %(message)s'),
'__force_dict__': True
}
}
}
service_logging = logging_template
if CONF.use_handlers == 'console':
del service_logging['handlers']['logfile']
elif CONF.use_handlers == 'Logfile':
del service_logging['handlers']['console']
return service_logging

View File

@ -2,9 +2,9 @@
# along with region, with each cms and fms api request
import orm.base_config as config
tenant_name = config.token_auth_tenant
username = config.token_auth_user
password = config.token_auth_pass
tenant_name = config.CONF.keystone_authtoken.project_name
username = config.CONF.keystone_authtoken.username
password = config.CONF.keystone_authtoken.password
auth_region = config.CONF.cli.base_region
orm_base_url = config.ranger_url
verify = config.ssl_verify
orm_base_url = config.CONF.ranger_url
verify = config.CONF.ssl_verify

View File

@ -3,7 +3,8 @@ import orm.base_config as config
# Server Specific Configurations
server = {
'port': config.audit_server['port'],
'host': config.orm_host
'host': config.orm_host,
'name': 'audit'
}
# DB configurations
@ -12,7 +13,7 @@ db_url = config.db_connect
database = {
'url': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_audit") or (db_url + 'orm_audit'),
'echo_statements': True
'echo_statements': False
}
# Pecan Application Configurations
@ -22,45 +23,8 @@ app = {
'debug': True,
}
logging = {
'root': {'level': 'INFO', 'handlers': ['console']},
'loggers': {
'orm.services.audit_trail_manager.audit_server': {
'level': config.debug_level,
'handlers': ['console', 'logfile'],
'propagate': False
},
'pecan': {'level': config.debug_level, 'handlers': ['console'],
'propagate': False},
'py.warnings': {'handlers': ['console']},
'__force_dict__': True
},
'handlers': {
'console': {
'level': config.debug_level,
'class': 'logging.StreamHandler',
'formatter': 'color'
},
'logfile': {
'level': config.debug_level,
'class': 'logging.FileHandler',
'formatter': 'color',
'filename': config.audit_server['log']
}
},
'formatters': {
'simple': {
'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]'
'[%(threadName)s] %(message)s')
},
'color': {
'()': 'pecan.log.ColorFormatter',
'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]'
'[%(threadName)s] %(message)s'),
'__force_dict__': True
}
}
}
app_module = app['modules'][0]
logging = config.get_log_config(config.audit_server['log'], server['name'], app_module)
verify = config.ssl_verify

View File

@ -186,5 +186,4 @@ class CustomerRecord:
if limit > 0:
query = query.limit(limit)
print str(query)
return query

View File

@ -87,5 +87,5 @@ class CustomerRegionRecord:
result = self.session.connection().execute(
"delete from customer_region where customer_id = {} and region_id <> -1 ".format(customer_id))
print "num records deleted from customer regions: " + str(result.rowcount)
# print "num records deleted from customer regions: " + str(result.rowcount)
return result

View File

@ -23,60 +23,8 @@ app = {
'hooks': lambda: [TransactionIdHook(), APIErrorHook(), SecurityHeadersHook()]
}
logging = {
'root': {'level': 'INFO', 'handlers': ['console']},
'loggers': {
'orm.services.customer_manager.cms_rest': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False
},
'orm.common.orm_common': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False
},
'orm.common.client.keystone.keystone_utils': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False
},
'pecan': {'level': 'DEBUG', 'handlers': ['console'],
'propagate': False},
'orm.common.client.audit.audit_client': {'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False},
'py.warnings': {'handlers': ['console']},
'__force_dict__': True
},
'handlers': {
'console': {
'level': config.debug_level,
'class': 'logging.StreamHandler',
'formatter': 'color'
},
'Logfile': {
'level': config.debug_level,
'class': 'logging.handlers.RotatingFileHandler',
'maxBytes': 50000000,
'backupCount': 10,
'filename': config.cms['log'],
'formatter': 'simple'
}
},
'formatters': {
'simple': {
'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]'
'[%(threadName)s] %(message)s')
},
'color': {
'()': 'pecan.log.ColorFormatter',
'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]'
'[%(threadName)s] %(message)s'),
'__force_dict__': True
}
}
}
app_module = app['modules'][0]
logging = config.get_log_config(config.cms['log'], server['name'], app_module)
quotas_default_values = {
'compute': {
@ -121,20 +69,4 @@ api = {
verify = config.ssl_verify
authentication = {
"enabled": config.token_auth_enabled,
"mech_id": config.token_auth_user,
"mech_pass": config.token_auth_pass,
"rms_url": config.rms['base_url'],
"tenant_name": config.token_auth_tenant,
"token_role": config.token_auth_user_role,
# The Keystone collection under which the role was granted.
# The key can be either "tenant" (for Keystone v2.0) or "domain"
# (for Keystone v3) and the value is the tenant/domain name.
"role_location": {"domain": "admin"},
# The Keystone version currently in use. Can be either "2.0" or "3".
"keystone_version": config.token_auth_version,
"policy_file": config.cms['policy_file'],
"user_domain_name": config.user_domain_name,
"project_domain_name": config.project_domain_name
}
authentication = config.server_request_auth(server['name'])

View File

@ -23,59 +23,8 @@ app = {
'hooks': lambda: [TransIdHook(), APIErrorHook(), SecurityHeadersHook()]
}
logging = {
'root': {'level': 'INFO', 'handlers': ['console']},
'loggers': {
'orm.services.flavor_manager.fms_rest': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False
},
'pecan': {'level': 'DEBUG', 'handlers': ['console'], 'propagate': False},
'orm.common.orm_common': {'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False},
'orm.common.client.keystone.keystone_utils': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False
},
'orm.common.client.audit.audit_client': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False
},
'py.warnings': {'handlers': ['console']},
'__force_dict__': True
},
'handlers': {
'console': {
'level': config.debug_level,
'class': 'logging.StreamHandler',
'formatter': 'color'
},
'Logfile': {
'level': config.debug_level,
'class': 'logging.handlers.RotatingFileHandler',
'maxBytes': 50000000,
'backupCount': 10,
'filename': config.fms['log'],
'formatter': 'simple'
}
},
'formatters': {
'simple': {
'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]'
'[%(threadName)s] %(message)s')
},
'color': {
'()': 'pecan.log.ColorFormatter',
'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]'
'[%(threadName)s] %(message)s'),
'__force_dict__': True
}
}
}
app_module = app['modules'][0]
logging = config.get_log_config(config.fms['log'], server['name'], app_module)
# DB configurations
db_url = config.db_connect
@ -148,22 +97,6 @@ api = {
}
verify = config.ssl_verify
authentication = {
"enabled": config.token_auth_enabled,
"mech_id": config.token_auth_user,
"mech_pass": config.token_auth_pass,
"rms_url": config.rms['base_url'],
"tenant_name": config.token_auth_tenant,
"token_role": config.token_auth_user_role,
# The Keystone version currently in use. Can be either "2.0" or "3"
"keystone_version": config.token_auth_version,
"policy_file": config.fms['policy_file'],
"user_domain_name": config.user_domain_name,
"project_domain_name": config.project_domain_name
}
# valid_flavor_options
flavor_options = {
'valid_cpin_opt_values': [
@ -192,3 +125,7 @@ flavor_limits = {
# ephemeral_limit is in GB
"ephemeral_limit": "10000"
}
verify = config.CONF.ssl_verify
authentication = config.server_request_auth(server['name'])

View File

@ -96,7 +96,6 @@ class FlavorRecord:
try:
query = self.session.query(Flavor)
query = self.customise_query(query, kw)
print str(query)
return query.all()
except Exception as exception:
@ -114,7 +113,6 @@ class FlavorRecord:
if limit > 0:
query = query.limit(limit)
print str(query)
return query
def get_count_of_flavors_by_series(self, series):

View File

@ -4,7 +4,8 @@ from orm import base_config as config
# Server Specific Configurations
server = {
'port': config.uuid['port'],
'host': config.orm_host
'host': config.orm_host,
'name': 'uuid'
}
# Pecan Application Configurations
app = {
@ -13,47 +14,8 @@ app = {
'debug': True,
}
logging = {
'root': {'level': 'INFO', 'handlers': ['console']},
'loggers': {
'orm.services.id_generator.uuidgen': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False
},
'pecan': {'level': config.debug_level, 'handlers': ['console'],
'propagate': False},
'py.warnings': {'handlers': ['console']},
'__force_dict__': True
},
'handlers': {
'console': {
'level': config.debug_level,
'class': 'logging.StreamHandler',
'formatter': 'color'
},
'Logfile': {
'level': config.debug_level,
'class': 'logging.handlers.RotatingFileHandler',
'maxBytes': 50000000,
'backupCount': 10,
'filename': config.uuid['log'],
'formatter': 'simple'
}
},
'formatters': {
'simple': {
'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]'
'[%(threadName)s] %(message)s')
},
'color': {
'()': 'pecan.log.ColorFormatter',
'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]'
'[%(threadName)s] %(message)s'),
'__force_dict__': True
}
}
}
app_module = app['modules'][0]
logging = config.get_log_config(config.uuid['log'], server['name'], app_module)
verify = config.ssl_verify

View File

@ -21,52 +21,8 @@ app = {
'hooks': lambda: [TransIdHook(), APIErrorHook(), SecurityHeadersHook()]
}
logging = {
'root': {'level': 'INFO', 'handlers': ['console']},
'loggers': {
'orm.services.image_manager.ims': {'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False},
'orm.common.client.audit.audit_client': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False},
'orm.common.orm_common': {'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False},
'pecan': {'level': config.debug_level, 'handlers': ['console'],
'propagate': False},
'py.warnings': {'handlers': ['console']},
'__force_dict__': True
},
'handlers': {
'console': {
'level': config.debug_level,
'class': 'logging.StreamHandler',
'formatter': 'color'
},
'Logfile': {
'level': config.debug_level,
'class': 'logging.handlers.RotatingFileHandler',
'maxBytes': 50000000,
'backupCount': 10,
'filename': config.ims['log'],
'formatter': 'simple'
}
},
'formatters': {
'simple': {
'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]'
'[%(threadName)s] %(message)s')
},
'color': {
'()': 'pecan.log.ColorFormatter',
'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]'
'[%(threadName)s] %(message)s'),
'__force_dict__': True
}
}
}
app_module = app['modules'][0]
logging = config.get_log_config(config.ims['log'], server['name'], app_module)
# DB configurations
db_url = config.db_connect
@ -103,15 +59,4 @@ api = {
verify = config.ssl_verify
authentication = {
"enabled": config.token_auth_enabled,
"mech_id": config.token_auth_user,
"mech_pass": config.token_auth_pass,
"rms_url": config.rms['base_url'],
"tenant_name": config.token_auth_tenant,
"token_role": config.token_auth_user_role,
"keystone_version": config.token_auth_version,
"policy_file": config.ims['policy_file'],
"user_domain_name": config.user_domain_name,
"project_domain_name": config.project_domain_name
}
authentication = config.server_request_auth(server['name'])

View File

@ -22,7 +22,6 @@ class Record(object):
if limit > 0:
query = query.limit(limit)
print str(query)
return query
# 5644 ProCG uses this line - don't edit it

View File

@ -21,60 +21,8 @@ app = {
'hooks': lambda: [TransactionIdHook(), APIErrorHook(), SecurityHeadersHook()]
}
logging = {
'root': {'level': 'INFO', 'handlers': ['console']},
'loggers': {
'orm.services.region_manager.rms': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False
},
'pecan': {'level': config.debug_level, 'handlers': ['console'],
'propagate': False},
'orm.common.client.audit.audit_client': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False
},
'orm.common.orm_common': {'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False},
'orm.common.client.keystone.keystone_utils': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False
},
'py.warnings': {'handlers': ['console']},
'__force_dict__': True
},
'handlers': {
'console': {
'level': config.debug_level,
'class': 'logging.StreamHandler',
'formatter': 'color'
},
'Logfile': {
'level': config.debug_level,
'class': 'logging.handlers.RotatingFileHandler',
'maxBytes': 50000000,
'backupCount': 10,
'filename': config.rms['log'],
'formatter': 'simple'
}
},
'formatters': {
'simple': {
'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]'
'[%(threadName)s] %(message)s')
},
'color': {
'()': 'pecan.log.ColorFormatter',
'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]'
'[%(threadName)s] %(message)s'),
'__force_dict__': True
}
}
}
app_module = app['modules'][0]
logging = config.get_log_config(config.rms['log'], server['name'], app_module)
# user input validations
region_options = {
@ -106,17 +54,7 @@ endpoints = {
verify = config.ssl_verify
authentication = {
"enabled": config.token_auth_enabled,
"mech_id": config.token_auth_user,
"mech_pass": config.token_auth_pass,
"tenant_name": config.token_auth_tenant,
# The Keystone version currently in use. Can be either "2.0" or "3"
"keystone_version": config.token_auth_version,
"policy_file": config.rms['policy_file'],
"user_domain_name": config.user_domain_name,
"project_domain_name": config.project_domain_name
}
authentication = config.server_request_auth(server['name'])
api = {
'uuid_server': {

View File

@ -3,7 +3,7 @@ import orm.base_config as config
app = {
'root': 'orm.services.resource_distributor.rds.controllers.root.RootController',
'modules': ['orm.services.resource_distributor.rds'],
'service_name': 'RDS'
'service_name': 'rds'
}
server = {
@ -159,66 +159,9 @@ region_resource_id_status = {
}
}
logging = {
'root': {'level': 'INFO', 'handlers': ['console']},
'loggers': {
'orm.services.resource_distributor.rds': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False
},
'orm.common.orm_common': {'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False},
'orm.common.client.audit.audit_client': {
'level': config.debug_level,
'handlers': ['console', 'Logfile'],
'propagate': False
},
'pecan': {'level': config.debug_level,
'handlers': ['console'], 'propagate': False},
'py.warnings': {'handlers': ['console']},
'__force_dict__': True
},
'handlers': {
'console': {
'level': config.debug_level,
'class': 'logging.StreamHandler',
'formatter': 'color'
},
'Logfile': {
'level': config.debug_level,
'class': 'logging.handlers.RotatingFileHandler',
'maxBytes': 50000000,
'backupCount': 10,
'filename': config.rds['log'],
'formatter': 'simple'
}
},
'formatters': {
'simple': {
'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]'
'[%(threadName)s] %(message)s')
},
'color': {
'()': 'pecan.log.ColorFormatter',
'format': '%(asctime)s [%(padded_color_levelname)s] [%(name)s] [%(threadName)s] %(message)s',
'__force_dict__': True
}
}
}
app_module = app['modules'][0]
logging = config.get_log_config(config.rds['log'], app['service_name'], app_module)
verify = config.ssl_verify
authentication = {
"enabled": config.token_auth_enabled,
"mech_id": config.token_auth_user,
"mech_pass": config.token_auth_pass,
"tenant_name": config.token_auth_tenant,
"token_role": config.token_auth_user_role,
# The Keystone version currently in use. Can be either "2.0" or "3"
"keystone_version": config.token_auth_version,
"user_domain_name": config.user_domain_name,
"project_domain_name": config.project_domain_name
}
authentication = config.server_request_auth(app['service_name'])