Switch freezer-scheduler to oslo.config and oslo.log

switch freezer-scheduler to use oslo.config and
switch from native python logging module to oslo.log
This commit includes:
- using oslo.config for parsing cli and config files options
- using oslo.log instead of native python logging module
- this applied only on freezer-scheduler
Implements: blueprint using-oslo-libs

Change-Id: I92e99c087cb2c2f836770644621f711af597dffc
This commit is contained in:
Saad Zaher 2015-12-02 19:37:36 +00:00
parent b6397636dc
commit 095142c40d
8 changed files with 267 additions and 240 deletions

22
HACKING.rst Normal file
View File

@ -0,0 +1,22 @@
Freezer Style Commandments
===========================
- Step 1: Read the OpenStack Style Commandments
http://docs.openstack.org/developer/hacking/
- Step 2: Read on
Freezer Specific Commandments
------------------------------
Logging
-------
Use the common logging module, and ensure you ``getLogger``::
from oslo_log import log
LOG = log.getLogger(__name__)
LOG.debug('Foobar')

View File

@ -0,0 +1,20 @@
# (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Freezer Versions
import pbr.version
__version__ = pbr.version.VersionInfo('freezer').version_string()

View File

@ -29,6 +29,9 @@ from registration import RegistrationManager
from jobs import JobManager from jobs import JobManager
from actions import ActionManager from actions import ActionManager
from sessions import SessionManager from sessions import SessionManager
from oslo_config import cfg
CONF = cfg.CONF
FREEZER_SERVICE_TYPE = 'backup' FREEZER_SERVICE_TYPE = 'backup'
@ -55,84 +58,82 @@ class cached_property(object):
return value return value
def build_os_option_parser(parser): def build_os_options():
parser.add_argument( osclient_opts = [
'--os-username', action='store', cfg.StrOpt('os-username',
help=('Name used for authentication with the OpenStack ' default=env('OS_USERNAME'),
'Identity service. Defaults to env[OS_USERNAME].'), help='Name used for authentication with the OpenStack '
dest='os_username', default=env('OS_USERNAME')) 'Identity service. Defaults to env[OS_USERNAME].',
parser.add_argument( dest='os_username'),
'--os-password', action='store', cfg.StrOpt('os-password',
help=('Password used for authentication with the OpenStack ' default=env('OS_PASSWORD'),
'Identity service. Defaults to env[OS_PASSWORD].'), help='Password used for authentication with the OpenStack '
dest='os_password', default=env('OS_PASSWORD')) 'Identity service. Defaults to env[OS_PASSWORD].',
parser.add_argument( dest='os_password'),
'--os-project-name', action='store', cfg.StrOpt('os-project-name',
help=('Project name to scope to. Defaults to ' default=env('OS_PROJECT_NAME'),
'env[OS_PROJECT_NAME].'), help='Project name to scope to. Defaults to '
dest='os_project_name', 'env[OS_PROJECT_NAME].',
default=env('OS_PROJECT_NAME', default='default')) dest='os_project_name'),
parser.add_argument( cfg.StrOpt('os-project-domain-name',
'--os-project-domain-name', action='store', default=env('OS_PROJECT_DOMAIN_NAME'),
help=('Domain name containing project. Defaults to ' help='Domain name containing project. Defaults to '
'env[OS_PROJECT_DOMAIN_NAME].'), 'env[OS_PROJECT_DOMAIN_NAME].',
dest='os_project_domain_name', default=env('OS_PROJECT_DOMAIN_NAME', dest='os_project_domain_name'),
default='default')) cfg.StrOpt('os-user-domain-name',
parser.add_argument( default=env('OS_USER_DOMAIN_NAME'),
'--os-user-domain-name', action='store', help='User\'s domain name. Defaults to '
help=('User\'s domain name. Defaults to ' 'env[OS_USER_DOMAIN_NAME].',
'env[OS_USER_DOMAIN_NAME].'), dest='os_user_domain_name'),
dest='os_user_domain_name', default=env('OS_USER_DOMAIN_NAME', cfg.StrOpt('os-tenant-name',
default='default')) default=env('OS_TENANT_NAME'),
parser.add_argument( help='Tenant to request authorization on. Defaults to '
'--os-tenant-name', action='store', 'env[OS_TENANT_NAME].',
help=('Tenant to request authorization on. Defaults to ' dest='os_tenant_name'),
'env[OS_TENANT_NAME].'), cfg.StrOpt('os-tenant-id',
dest='os_tenant_name', default=env('OS_TENANT_NAME')) default=env('OS_TENANT_ID'),
parser.add_argument( help='Tenant to request authorization on. Defaults to '
'--os-tenant-id', action='store', 'env[OS_TENANT_ID].',
help=('Tenant to request authorization on. Defaults to ' dest='os_tenant_id'),
'env[OS_TENANT_ID].'), cfg.StrOpt('os-auth-url',
dest='os_tenant_id', default=env('OS_TENANT_ID')) default=env('OS_AUTH_URL'),
parser.add_argument( help='Specify the Identity endpoint to use for '
'--os-auth-url', action='store', 'authentication. Defaults to env[OS_AUTH_URL].',
help=('Specify the Identity endpoint to use for ' dest='os_auth_url'),
'authentication. Defaults to env[OS_AUTH_URL].'), cfg.StrOpt('os-backup-url',
dest='os_auth_url', default=env('OS_AUTH_URL')) default=env('OS_BACKUP_URL'),
parser.add_argument( help='Specify the Freezer backup service endpoint to use. '
'--os-backup-url', action='store', 'Defaults to env[OS_BACKUP_URL].',
help=('Specify the Freezer backup service endpoint to use. ' dest='os_backup_url'),
'Defaults to env[OS_BACKUP_URL].'), cfg.StrOpt('os-region-name',
dest='os_backup_url', default=env('OS_BACKUP_URL')) default=env('OS_REGION_NAME'),
parser.add_argument( help='Specify the region to use. Defaults to '
'--os-region-name', action='store', 'env[OS_REGION_NAME].',
help=('Specify the region to use. Defaults to ' dest='os_region_name'),
'env[OS_REGION_NAME].'), cfg.StrOpt('os-token',
dest='os_region_name', default=env('OS_REGION_NAME')) default=env('OS_TOKEN'),
parser.add_argument( help='Specify an existing token to use instead of retrieving'
'--os-token', action='store', ' one via authentication (e.g. with username & '
help=('Specify an existing token to use instead of retrieving' 'password). Defaults to env[OS_TOKEN].',
' one via authentication (e.g. with username & password). ' dest='os_token'),
'Defaults to env[OS_TOKEN].'), cfg.StrOpt('os-identity-api-version',
dest='os_token', default=env('OS_TOKEN')) default=env('OS_IDENTITY_API_VERSION'),
parser.add_argument( help='Identity API version: 2.0 or 3. '
'--os-identity-api-version', action='store', 'Defaults to env[OS_IDENTITY_API_VERSION]',
help=('Identity API version: 2.0 or 3. ' dest='os_identity_api_version'),
'Defaults to env[OS_IDENTITY_API_VERSION]'), cfg.StrOpt('os-endpoint-type',
dest='os_identity_api_version', choices=['public', 'publicURL', 'internal', 'internalURL',
default=env('OS_IDENTITY_API_VERSION')) 'admin', 'adminURL'],
parser.add_argument( default=env('OS_ENDPOINT_TYPE') or 'public',
'--os-endpoint-type', action='store', help='Endpoint type to select. Valid endpoint types: '
choices=['public', 'publicURL', 'internal', 'internalURL', '"public" or "publicURL", "internal" or "internalURL",'
'admin', 'adminURL'], ' "admin" or "adminURL". Defaults to '
help=('Endpoint type to select. ' 'env[OS_ENDPOINT_TYPE] or "public"',
'Valid endpoint types: "public" or "publicURL", ' dest='os_endpoint_type'),
'"internal" or "internalURL", "admin" or "adminURL". '
'Defaults to env[OS_ENDPOINT_TYPE] or "public"'),
dest='os_endpoint_type',
default=env('OS_ENDPOINT_TYPE', default='public'))
return parser ]
return osclient_opts
def guess_auth_version(opts): def guess_auth_version(opts):
@ -193,9 +194,7 @@ class Client(object):
project_domain_name=None, project_domain_name=None,
verify=True): verify=True):
self.opts = opts or build_os_option_parser( self.opts = opts
argparse.ArgumentParser(description='Freezer Client')
).parse_known_args()[0]
if token: if token:
self.opts.os_token = token self.opts.os_token = token
if username: if username:

View File

@ -17,6 +17,14 @@ limitations under the License.
import argparse import argparse
import os import os
from oslo_config import cfg
from oslo_log import log
from freezer import __version__ as FREEZER_VERSION
import sys
CONF = cfg.CONF
_LOG = log.getLogger(__name__)
from freezer.apiclient import client as api_client from freezer.apiclient import client as api_client
from freezer import winutils from freezer import winutils
@ -27,88 +35,102 @@ else:
DEFAULT_FREEZER_SCHEDULER_CONF_D = '/etc/freezer/scheduler/conf.d' DEFAULT_FREEZER_SCHEDULER_CONF_D = '/etc/freezer/scheduler/conf.d'
def base_parser(parser): def getCommonOpts():
scheduler_conf_d = os.environ.get('FREEZER_SCHEDULER_CONF_D', scheduler_conf_d = os.environ.get('FREEZER_SCHEDULER_CONF_D',
DEFAULT_FREEZER_SCHEDULER_CONF_D) DEFAULT_FREEZER_SCHEDULER_CONF_D)
parser.add_argument(
'-j', '--job', action='store',
help=('name or ID of the job'),
dest='job_id', default=None)
parser.add_argument(
'-s', '--session', action='store',
help=('name or ID of the session'),
dest='session_id', default=None)
parser.add_argument(
'--file', action='store',
help=('Local file that contains the resource '
'to be uploaded/downloaded'),
dest='fname', default=None)
parser.add_argument(
'-c', '--client-id', action='store',
help=('Specifies the client_id used when contacting the service.'
'If not specified it will be automatically created'
'using the tenant-id and the machine hostname.'),
dest='client_id', default=None)
parser.add_argument(
'-n', '--no-api', action='store_true',
help='Prevents the scheduler from using the api service',
dest='no_api', default=False)
parser.add_argument(
'-a', '--active-only', action='store_true',
help='Filter only active jobs/session',
dest='active_only', default=False)
parser.add_argument(
'-f', '--conf', action='store',
help=('Used to store/retrieve files on local storage, including '
'those exchanged with the api service. '
'Default value is {0} '
'(Env: FREEZER_SCHEDULER_CONF_D)'.format(scheduler_conf_d)),
dest='jobs_dir', default=scheduler_conf_d)
parser.add_argument(
'-i', '--interval', action='store',
help=('Specifies the api-polling interval in seconds.'
'Defaults to 60 seconds'),
dest='interval', default=60)
parser.add_argument( common_opts = [
'-v', '--verbose', cfg.StrOpt('job',
action='count', default=None,
dest='verbose_level', dest='job_id',
default=1, short='j',
help='Increase verbosity of output. Can be repeated.', help='Name or ID of the job'),
) cfg.StrOpt('session',
parser.add_argument( default=None,
'--debug', dest='session_id',
default=False, short='s',
action='store_true', help='Name or ID of the session'),
help='show tracebacks on errors', cfg.StrOpt('file',
) default=None,
parser.add_argument( dest='fname',
'--no-daemon', help='Local file that contains the resource to be '
action='store_true', 'uploaded/downloaded'),
help='Prevents the scheduler from running in daemon mode', cfg.StrOpt('client-id',
dest='no_daemon', default=False default=None,
) dest='client_id',
parser.add_argument( short='c',
'-l', '--log-file', action='store', help='Specifies the client_id used when contacting the service.'
help=('location of log file'), '\n If not specified it will be automatically created \n'
dest='log_file', default=None) 'using the tenant-id and the machine hostname.'),
cfg.BoolOpt('no-api',
default=False,
dest='no_api',
short='n',
help='Prevents the scheduler from using the api service'),
cfg.BoolOpt('active-only',
default=False,
dest='active_only',
short='a',
help='Filter only active jobs/session'),
cfg.StrOpt('conf',
default=scheduler_conf_d,
dest='jobs_dir',
short='f',
help='Used to store/retrieve files on local storage, including '
'those exchanged with the api service.Default value is {0} '
'(Env: FREEZER_SCHEDULER_CONF_D)'.format(scheduler_conf_d)),
cfg.IntOpt('interval',
default=60,
dest='interval',
short='i',
help='Specifies the api-polling interval in seconds. '
'Defaults to 60 seconds'),
cfg.BoolOpt('no-daemon',
default=False,
dest='no_daemon',
help='Prevents the scheduler from running in daemon mode'),
cfg.BoolOpt('insecure',
default=False,
dest='insecure',
help='Initialize freezer scheduler with insecure mode'),
]
parser.add_argument( return common_opts
'--insecure',
action='store_true',
help='Initialize freezer scheduler with insecure mode',
dest='insecure', default=False
)
return parser
def get_args(choices): def parse_args(choices):
parser = base_parser( default_conf = cfg.find_config_files('freezer', 'freezer-scheduler',
api_client.build_os_option_parser( '.conf')
argparse.ArgumentParser(description='Freezer Scheduler') CONF.register_cli_opts(api_client.build_os_options())
)) CONF.register_cli_opts(getCommonOpts())
parser.add_argument( log.register_options(CONF)
'action', action='store', default=None, choices=choices, help='')
return parser.parse_args() positional = [
cfg.StrOpt('action',
choices=choices,
default=None,
help='{0}'.format(choices), positional=True),
]
CONF.register_cli_opts(positional)
CONF(args=sys.argv[1:],
project='freezer-scheduler',
default_config_files=default_conf,
version=FREEZER_VERSION
)
def setup_logging():
_DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
'qpid=WARN', 'stevedore=WARN',
'oslo_log=INFO', 'iso8601=WARN',
'requests.packages.urllib3.connectionpool=WARN',
'urllib3.connectionpool=WARN', 'websocket=WARN',
'keystonemiddleware=WARN', 'freezer=INFO']
_DEFAULT_LOGGING_CONTEXT_FORMAT = ('%(asctime)s.%(msecs)03d %(process)d '
'%(levelname)s %(name)s [%(request_id)s '
'%(user_identity)s] %(instance)s'
'%(message)s')
log.set_defaults(_DEFAULT_LOGGING_CONTEXT_FORMAT, _DEFAULT_LOG_LEVELS)
log.setup(CONF, 'freezer-scheduler', version=FREEZER_VERSION)

View File

@ -15,8 +15,6 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
""" """
import logging
import os import os
import sys import sys
import threading import threading
@ -36,17 +34,24 @@ if winutils.is_windows():
else: else:
from daemon import Daemon, NoDaemon from daemon import Daemon, NoDaemon
from scheduler_job import Job from scheduler_job import Job
from oslo_config import cfg
from oslo_log import log
CONF = cfg.CONF
LOG = log.getLogger(__name__)
class FreezerScheduler(object): class FreezerScheduler(object):
def __init__(self, apiclient, interval, job_path): def __init__(self, apiclient, interval, job_path):
# config_manager # config_manager
self.client = apiclient self.client = apiclient
self.freezerc_executable = spawn.find_executable('freezerc') self.freezerc_executable = spawn.find_executable('freezer-agent')
if self.freezerc_executable is None: if self.freezerc_executable is None:
# Needed in the case of a non-activated virtualenv # Needed in the case of a non-activated virtualenv
self.freezerc_executable = spawn.find_executable( self.freezerc_executable = spawn.find_executable(
'freezerc', path=':'.join(sys.path)) 'freezer-agent', path=':'.join(sys.path))
LOG.debug('Freezer-agent found at {0}'.format(self.freezerc_executable))
self.job_path = job_path self.job_path = job_path
self._client = None self._client = None
self.lock = threading.Lock() self.lock = threading.Lock()
@ -70,7 +75,7 @@ class FreezerScheduler(object):
try: try:
utils.save_jobs_to_disk(job_doc_list, self.job_path) utils.save_jobs_to_disk(job_doc_list, self.job_path)
except Exception as e: except Exception as e:
logging.error('Unable to save jobs to {0}. ' LOG.error('Unable to save jobs to {0}. '
'{1}'.format(self.job_path, e)) '{1}'.format(self.job_path, e))
return job_doc_list return job_doc_list
else: else:
@ -107,7 +112,7 @@ class FreezerScheduler(object):
try: try:
return self.client.jobs.update(job_id, job_doc) return self.client.jobs.update(job_id, job_doc)
except Exception as e: except Exception as e:
logging.error("[*] Job update error: {0}".format(e)) LOG.error("[*] Job update error: {0}".format(e))
def update_job_status(self, job_id, status): def update_job_status(self, job_id, status):
doc = {'job_schedule': {'status': status}} doc = {'job_schedule': {'status': status}}
@ -120,14 +125,14 @@ class FreezerScheduler(object):
job = Job.create(self, self.freezerc_executable, job_doc) job = Job.create(self, self.freezerc_executable, job_doc)
if job: if job:
self.jobs[job.id] = job self.jobs[job.id] = job
logging.info("Created job {0}".format(job.id)) LOG.info("Created job {0}".format(job.id))
return job return job
def poll(self): def poll(self):
try: try:
work_job_doc_list = self.get_jobs() work_job_doc_list = self.get_jobs()
except Exception as e: except Exception as e:
logging.error("[*] Unable to get jobs: {0}".format(e)) LOG.error("[*] Unable to get jobs: {0}".format(e))
return return
work_job_id_list = [] work_job_id_list = []
@ -158,7 +163,7 @@ class FreezerScheduler(object):
pass pass
def reload(self): def reload(self):
logging.warning("reload not supported") LOG.warning("reload not supported")
def _get_doers(module): def _get_doers(module):
@ -176,56 +181,64 @@ def main():
possible_actions = doers.keys() + ['start', 'stop', 'status'] possible_actions = doers.keys() + ['start', 'stop', 'status']
args = arguments.get_args(possible_actions) arguments.parse_args(possible_actions)
arguments.setup_logging()
if args.action is None: if CONF.action is None:
print ('No action') CONF.print_help()
return 65 # os.EX_DATAERR return 65 # os.EX_DATAERR
apiclient = None apiclient = None
verify = True verify = True
if args.insecure: if CONF.insecure:
verify = False verify = False
if args.no_api is False: if CONF.no_api is False:
apiclient = client.Client(opts=args, verify=verify) try:
if args.client_id: apiclient = client.Client(opts=CONF, verify=verify)
apiclient.client_id = args.client_id if CONF.client_id:
apiclient.client_id = CONF.client_id
except Exception as e:
LOG.error(e)
print e
sys.exit(1)
else: else:
if winutils.is_windows(): if winutils.is_windows():
print("--no-api mode is not available on windows") print("--no-api mode is not available on windows")
return 69 # os.EX_UNAVAILABLE return 69 # os.EX_UNAVAILABLE
if args.action in doers: if CONF.action in doers:
try: try:
return doers[args.action](apiclient, args) return doers[CONF.action](apiclient, CONF)
except Exception as e: except Exception as e:
LOG.error(e)
print ('ERROR {0}'.format(e)) print ('ERROR {0}'.format(e))
return 70 # os.EX_SOFTWARE return 70 # os.EX_SOFTWARE
freezer_scheduler = FreezerScheduler(apiclient=apiclient, freezer_scheduler = FreezerScheduler(apiclient=apiclient,
interval=int(args.interval), interval=int(CONF.interval),
job_path=args.jobs_dir) job_path=CONF.jobs_dir)
if args.no_daemon: if CONF.no_daemon:
print ('Freezer Scheduler running in no-daemon mode') print ('Freezer Scheduler running in no-daemon mode')
LOG.debug('Freezer Scheduler running in no-daemon mode')
daemon = NoDaemon(daemonizable=freezer_scheduler) daemon = NoDaemon(daemonizable=freezer_scheduler)
else: else:
if winutils.is_windows(): if winutils.is_windows():
daemon = Daemon(daemonizable=freezer_scheduler, daemon = Daemon(daemonizable=freezer_scheduler,
interval=int(args.interval), interval=int(CONF.interval),
job_path=args.jobs_dir, job_path=CONF.jobs_dir,
insecure=args.insecure) insecure=CONF.insecure)
else: else:
daemon = Daemon(daemonizable=freezer_scheduler) daemon = Daemon(daemonizable=freezer_scheduler)
if args.action == 'start': if CONF.action == 'start':
daemon.start(log_file=args.log_file) daemon.start(log_file=CONF.log_file)
elif args.action == 'stop': elif CONF.action == 'stop':
daemon.stop() daemon.stop()
elif args.action == 'reload': elif CONF.action == 'reload':
daemon.reload() daemon.reload()
elif args.action == 'status': elif CONF.action == 'status':
daemon.status() daemon.status()
# os.RETURN_CODES are only available to posix like systems, on windows # os.RETURN_CODES are only available to posix like systems, on windows

View File

@ -9,6 +9,8 @@ python-novaclient>=2.28.1,!=2.33.0
python-openstackclient>=1.5.0 python-openstackclient>=1.5.0
oslo.utils>=2.0.0,!=2.6.0 # Apache-2.0 oslo.utils>=2.0.0,!=2.6.0 # Apache-2.0
oslo.i18n>=1.5.0 # Apache-2.0 oslo.i18n>=1.5.0 # Apache-2.0
oslo.log>=1.8.0 # Apache-2.0
oslo.config>=2.3.0 # Apache-2.0
PyMySQL>=0.6.2 # MIT License PyMySQL>=0.6.2 # MIT License
pymongo>=3.0.2 pymongo>=3.0.2

View File

@ -35,17 +35,6 @@ class TestSupportFunctions(unittest.TestCase):
var = client.env('TEST_ENV_VAR') var = client.env('TEST_ENV_VAR')
self.assertEquals(var, '') self.assertEquals(var, '')
@patch('freezer.apiclient.client.env')
def test_build_os_option_parser(self, mock_env):
mock_env.return_value = ''
mock_parser = Mock()
mock_parser._me = 'test12345'
retval = client.build_os_option_parser(mock_parser)
self.assertEquals(retval._me, 'test12345')
call_count = mock_parser.add_argument.call_count
self.assertGreater(call_count, 10)
def test_guess_auth_version_returns_none(self): def test_guess_auth_version_returns_none(self):
mock_opts = Mock() mock_opts = Mock()
mock_opts.os_identity_api_version = '' mock_opts.os_identity_api_version = ''

View File

@ -1,40 +0,0 @@
# (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from mock import Mock, patch
from freezer.scheduler import arguments
class TestBaseParser(unittest.TestCase):
def test_returns_parser(self):
mock_parser = Mock()
mock_parser._me = 'test12345'
retval = arguments.base_parser(mock_parser)
self.assertEquals(retval._me, 'test12345')
call_count = mock_parser.add_argument.call_count
self.assertGreater(call_count, 10)
@patch('freezer.scheduler.arguments.base_parser')
def test_get_args_return_parsed_args(self, mock_base_parser):
mock_parser = Mock()
mock_parser.parse_args.return_value = 'pluto'
mock_base_parser.return_value = mock_parser
retval = arguments.get_args(['alpha', 'bravo'])
call_count = mock_parser.add_argument.call_count
self.assertGreater(call_count, 0)