openstack
/
trove
Code Issues Proposed changes

More changes to facilitate oslo. 

* Cleaning up the CONF defaults in paste created classes
* Updated mgmt-taskmanager
* Small updates to files for oslo
* Removed executils, local
* Fixing constants (removing them from inside method defs)

implements blueprint reddwarf/upgrade-oslo

Change-Id: Ib1a9fe4a829ce2541323693650db412209298a9f
This commit is contained in:
Michael Basnight 2012-12-19 15:03:34 -06:00
parent ab6bafdb5a
commit 4eb7e34ca9
16 changed files with 49 additions and 429 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
bin/reddwarf-manage
View File

@ -39,6 +39,7 @@ from reddwarf.common import cfg
from reddwarf.common import utils
from reddwarf.db import get_db_api
from reddwarf.openstack.common import log as logging
from reddwarf.openstack.common import uuidutils
from reddwarf.instance import models as instance_models
@ -75,7 +76,7 @@ class Commands(object):
if image is None:
# Create a new one
image = instance_models.ServiceImage()
image.id = utils.generate_uuid()
image.id = uuidutils.generate_uuid()
image.service_name = service_name
image.image_id = image_id
self.db_api.save(image)

31
bin/reddwarf-mgmt-taskmanager
View File

@ -36,29 +36,26 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
if os.path.exists(os.path.join(possible_topdir, 'reddwarf', '__init__.py')):
sys.path.insert(0, possible_topdir)
from reddwarf import version
from reddwarf.common import config
from reddwarf.common import service
from reddwarf.common import cfg
from reddwarf.common import rpc
from reddwarf.openstack.common import cfg as openstack_cfg
from reddwarf.openstack.common import log as logging
from reddwarf.openstack.common import service
from reddwarf.db import get_db_api
CONF = cfg.CONF
CONF.register_opts([openstack_cfg.StrOpt('taskmanager_manager')])
if __name__ == '__main__':
parser = optparse.OptionParser(version="%%prog %s"
% version.version_string())
config.add_common_options(parser)
config.add_log_options(parser)
cfg.parse_args(sys.argv)
logging.setup(None)
(options, args) = config.parse_options(parser)
try:
conf, app = config.Config.load_paste_app('reddwarf-taskmanager',
options, args)
get_db_api().configure_db(conf)
server = service.Service.create(binary='reddwarf-taskmanager',
manager='reddwarf.taskmanager.manager.TaskManager',
topic="mgmt-taskmanager")
service.serve(server)
service.wait()
get_db_api().configure_db(CONF)
server = rpc.RpcService(manager=CONF.taskmanager_manager,
topic="mgmt-taskmanager")
launcher = service.launch(server)
launcher.wait()
except RuntimeError as error:
import traceback
print traceback.format_exc()

8
reddwarf/common/api.py
View File

@ -48,14 +48,6 @@ class API(wsgi.Router):
path = "/{tenant_id}/flavors"
mapper.resource("flavor", path, controller=flavor_resource)
def _host_instance_router(self, mapper):
host_controller = hostservice.HostInstanceController()
host_instance_resource = host_controller.create_resource()
path = "/{tenant_id}/mgmt/hosts/{host_id}/instances"
mapper.resource("hostinstance", path,
controller=host_instance_resource,
member={'action': 'POST'})
def app_factory(global_conf, **local_conf):
return API()

3
reddwarf/common/cfg.py
View File

@ -34,7 +34,7 @@ common_opts = [
default=False,
help='Whether to add IP addresses to the list operations'),
cfg.BoolOpt('reddwarf_volume_support',
default=False,
default=True,
help='File name for the paste.deploy config for reddwarf-api'),
cfg.ListOpt('admin_roles', default=[]),
cfg.StrOpt('remote_implementation',
@ -87,6 +87,7 @@ common_opts = [
cfg.IntOpt('server_delete_time_out', default=2),
cfg.IntOpt('volume_time_out', default=2),
cfg.IntOpt('reboot_time_out', default=60 * 2),
cfg.StrOpt('service_options', default=['mysql']),
]

48
reddwarf/common/excutils.py
View File

@ -1,48 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 OpenStack LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Exception related utilities.
"""
import contextlib
from reddwarf.openstack.common import log as logging
import sys
import traceback
@contextlib.contextmanager
def save_and_reraise_exception():
"""Save current exception, run some code and then re-raise.
In some cases the exception context can be cleared, resulting in None
being attempted to be reraised after an exception handler is run. This
can happen when eventlet switches greenthreads or when running an
exception handler, code raises and catches an exception. In both
cases the exception context will be cleared.
To work around this, we save the exception state, run handler code, and
then re-raise the original exception. If another exception occurs, the
saved exception is logged and the new exception is reraised.
"""
type_, value, tb = sys.exc_info()
try:
yield
except Exception:
logging.error('Original exception being dropped: %s' %
(traceback.format_exception(type_, value, tb)))
raise
raise type_, value, tb

37
reddwarf/common/local.py
View File

@ -1,37 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Greenthread local storage of variables using weak references"""
import weakref
from eventlet import corolocal
class WeakLocal(corolocal.local):
def __getattribute__(self, attr):
rval = corolocal.local.__getattribute__(self, attr)
if rval:
rval = rval()
return rval
def __setattr__(self, attr, value):
value = weakref.ref(value)
return corolocal.local.__setattr__(self, attr, value)
store = WeakLocal()

8
reddwarf/common/remote.py
View File

@ -20,6 +20,10 @@ from novaclient.v1_1.client import Client
CONF = cfg.CONF
COMPUTE_URL = CONF.nova_compute_url
PROXY_AUTH_URL = CONF.reddwarf_auth_url
VOLUME_URL = CONF.nova_volume_url
PROXY_AUTH_URL = CONF.reddwarf_auth_url
def create_dns_client(context):
@ -33,8 +37,6 @@ def create_guest_client(context, id):
def create_nova_client(context):
COMPUTE_URL = CONF.nova_compute_url
PROXY_AUTH_URL = CONF.reddwarf_auth_url
client = Client(context.user, context.auth_tok, project_id=context.tenant,
auth_url=PROXY_AUTH_URL)
client.client.auth_token = context.auth_tok
@ -46,8 +48,6 @@ def create_nova_client(context):
def create_nova_volume_client(context):
# Quite annoying but due to a paste config loading bug.
# TODO(hub-cap): talk to the openstack-common people about this
VOLUME_URL = CONF.nova_volume_url
PROXY_AUTH_URL = CONF.reddwarf_auth_url
client = Client(context.user, context.auth_tok,
project_id=context.tenant, auth_url=PROXY_AUTH_URL)
client.client.auth_token = context.auth_tok

241
reddwarf/common/service.py
View File

@ -1,241 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""STOLEN FROM NOVA."""
import functools
import inspect
import os
import socket
import traceback
import weakref
import eventlet
import greenlet
from eventlet import greenthread
from reddwarf import version
from reddwarf.common import cfg
from reddwarf.common import utils
from reddwarf.openstack.common import log as logging
from reddwarf.openstack.common import rpc
from reddwarf.openstack.common.gettextutils import _
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class Launcher(object):
"""Launch one or more services and wait for them to complete."""
def __init__(self):
"""Initialize the service launcher."""
self._services = []
@staticmethod
def run_server(server):
"""Start and wait for a server to finish."""
server.start()
server.wait()
def launch_server(self, server):
"""Load and start the given server."""
gt = eventlet.spawn(self.run_server, server)
self._services.append(gt)
def stop(self):
"""Stop all services which are currently running."""
for service in self._services:
service.kill()
def wait(self):
"""Waits until all services have been stopped, and then returns."""
for service in self._services:
try:
service.wait()
except greenlet.GreenletExit:
LOG.error(_("greenthread exited"))
pass
class Service(object):
"""Generic code to start services and get them listening on rpc"""
def __init__(self, host, binary, topic, manager, report_interval=None,
periodic_interval=None, *args, **kwargs):
if not host:
host = socket.gethostname()
self.host = host
self.binary = binary
self.topic = topic
self.manager_class_name = manager
manager_class = utils.import_class(self.manager_class_name)
self.manager = manager_class(host=self.host, *args, **kwargs)
self.report_interval = report_interval
self.periodic_interval = periodic_interval
super(Service, self).__init__(*args, **kwargs)
self.saved_args, self.saved_kwargs = args, kwargs
self.timers = []
def dispatch(self, ctxt, version, method, **kwargs):
"""Handles incoming RPC messages."""
#TODO(tim.simpson): Maybe in the future actually account for the
# version somehow with multiple managers or by
# sending the manager in or something.
if not version:
version = '1.0'
if version != self.manager.RPC_API_VERSION:
raise rpc.common.UnsupportedRpcVersion(version=version)
return self.manager.wrapper(method, ctxt, **kwargs)
def periodic_tasks(self, raise_on_error=False):
"""Tasks to be run at a periodic interval."""
self.manager.periodic_tasks(raise_on_error=raise_on_error)
def report_state(self):
pass
def start(self):
vcs_string = version.version_string_with_vcs()
LOG.info(_('Starting %(topic)s node (version %(vcs_string)s)'),
{'topic': self.topic, 'vcs_string': vcs_string})
self.conn = rpc.create_connection(new=True)
LOG.debug(_("Creating Consumer connection for Service %s") %
self.topic)
# Share this same connection for these Consumers
self.conn.create_consumer(self.topic, self, fanout=False)
node_topic = '%s.%s' % (self.topic, self.host)
self.conn.create_consumer(node_topic, self, fanout=False)
self.conn.create_consumer(self.topic, self, fanout=True)
# Consume from all consumers in a thread
self.conn.consume_in_thread()
if self.report_interval:
pulse = utils.LoopingCall(self.report_state)
pulse.start(interval=self.report_interval, now=False)
self.timers.append(pulse)
if self.periodic_interval:
periodic = utils.LoopingCall(self.periodic_tasks)
periodic.start(interval=self.periodic_interval, now=False)
self.timers.append(periodic)
def wait(self):
for x in self.timers:
try:
x.wait()
except Exception:
pass
@classmethod
def create(cls, host=None, binary=None, topic=None, manager=None,
report_interval=None, periodic_interval=None):
"""Instantiates class and passes back application object.
:param host: defaults to FLAGS.host
:param binary: defaults to basename of executable
:param topic: defaults to bin_name - 'nova-' part
:param manager: defaults to FLAGS.<topic>_manager
:param report_interval: defaults to FLAGS.report_interval
:param periodic_interval: defaults to FLAGS.periodic_interval
"""
if not host:
host = CONF.host
if not binary:
binary = os.path.basename(inspect.stack()[-1][1])
if not topic:
topic = binary.rpartition('reddwarf-')[2]
if not manager:
manager = CONF._get('%s_manager' % topic)
if not report_interval:
report_interval = CONF.report_interval
if not periodic_interval:
periodic_interval = CONF.periodic_interval
service_obj = cls(host, binary, topic, manager, report_interval,
periodic_interval)
return service_obj
class Manager(object):
def __init__(self, host=None):
self.host = host
self.tasks = weakref.WeakKeyDictionary()
super(Manager, self).__init__()
def periodic_tasks(self, raise_on_error=False):
"""Tasks to be run at a periodic interval."""
LOG.debug("No. of running tasks: %d" % len(self.tasks))
def init_host(self):
"""Handle initialization if this is a standalone service.
Child classes should override this method.
"""
pass
#TODO(tim.simpson): Rename this to "execute" or something clearer.
def wrapper(self, method, context, *args, **kwargs):
"""Maps the respective manager method with a task counter."""
# TODO(rnirmal): Just adding a basic counter. Will revist and
# re-implement when we have actual tasks.
self.tasks[greenthread.getcurrent()] = context
try:
if not hasattr(self, method):
raise AttributeError("No such RPC function '%s'" % method)
func = getattr(self, method)
LOG.info(str('*' * 80))
LOG.info("Running method %s..." % method)
LOG.info(str('*' * 80))
result = func(context, *args, **kwargs)
LOG.info("Finished method %s." % method)
return result
except Exception as e:
LOG.error("Got an error running %s!" % method)
LOG.error(traceback.format_exc())
finally:
LOG.info(str('-' * 80))
del self.tasks[greenthread.getcurrent()]
_launcher = None
def serve(*servers):
global _launcher
if not _launcher:
_launcher = Launcher()
for server in servers:
_launcher.launch_server(server)
def wait():
try:
_launcher.wait()
except KeyboardInterrupt:
_launcher.stop()
rpc.cleanup()

4
reddwarf/common/wsgi.py
View File

@ -331,10 +331,6 @@ class Controller(object):
],
}
def __init__(self):
self.add_addresses = CONF.add_addresses
self.add_volumes = CONF.reddwarf_volume_support
def create_resource(self):
serializer = ReddwarfResponseSerializer(
body_serializers={'application/xml': ReddwarfXMLDictSerializer()})

6
reddwarf/extensions/mgmt/instances/service.py
View File

@ -59,9 +59,7 @@ class MgmtInstanceController(InstanceController):
return wsgi.Result(str(e), 403)
view_cls = views.MgmtInstancesView
return wsgi.Result(view_cls(instances, req=req,
add_addresses=self.add_addresses,
add_volumes=self.add_volumes).data(), 200)
return wsgi.Result(view_cls(instances, req=req).data(), 200)
@admin_context
def show(self, req, tenant_id, id):
@ -78,8 +76,6 @@ class MgmtInstanceController(InstanceController):
views.MgmtInstanceDetailView(
server,
req=req,
add_addresses=self.add_addresses,
add_volumes=self.add_volumes,
root_history=root_history).data(),
200)

24
reddwarf/extensions/mgmt/instances/views.py
View File

@ -21,11 +21,8 @@ from reddwarf.instance.views import InstanceDetailView
class MgmtInstanceView(InstanceDetailView):
def __init__(self, instance, req=None, add_addresses=False,
add_volumes=False):
super(MgmtInstanceView, self).__init__(instance, req,
add_addresses,
add_volumes)
def __init__(self, instance, req=None):
super(MgmtInstanceView, self).__init__(instance, req)
def data(self):
result = super(MgmtInstanceView, self).data()
@ -59,13 +56,9 @@ class MgmtInstanceView(InstanceDetailView):
class MgmtInstanceDetailView(MgmtInstanceView):
"""Works with a full-blown instance."""
def __init__(self, instance, req, add_addresses=False,
add_volumes=False, root_history=None):
add_a = add_addresses
def __init__(self, instance, req, root_history=None):
super(MgmtInstanceDetailView, self).__init__(instance,
req=req,
add_addresses=add_a,
add_volumes=add_volumes)
req=req)
self.root_history = root_history
def data(self):
@ -102,12 +95,9 @@ class MgmtInstanceDetailView(MgmtInstanceView):
class MgmtInstancesView(object):
"""Shows a list of MgmtInstance objects."""
def __init__(self, instances, req=None, add_addresses=False,
add_volumes=False):
def __init__(self, instances, req=None):
self.instances = instances
self.req = req
self.add_addresses = add_addresses
self.add_volumes = add_volumes
def data(self):
data = []
@ -117,9 +107,7 @@ class MgmtInstancesView(object):
return {'instances': data}
def data_for_instance(self, instance):
view = MgmtInstanceView(instance, req=self.req,
add_addresses=self.add_addresses,
add_volumes=self.add_volumes)
view = MgmtInstanceView(instance, req=self.req)
return view.data()['instance']

3
reddwarf/instance/models.py
View File

@ -413,8 +413,7 @@ class Instance(BuiltInstance):
instance_id=db_info.id,
status=ServiceStatuses.NEW)
dns_support = CONF.reddwarf_dns_support
if dns_support:
if CONF.reddwarf_dns_support:
dns_client = create_dns_client(context)
hostname = dns_client.determine_hostname(db_info.id)
db_info.hostname = hostname

30
reddwarf/instance/service.py
View File

@ -136,8 +136,7 @@ class InstanceController(wsgi.Controller):
LOG.info(_("Indexing a database instance for tenant '%s'") % tenant_id)
context = req.environ[wsgi.CONTEXT_KEY]
servers, marker = models.Instances.load(context)
view = views.InstancesView(servers, req=req,
add_volumes=self.add_volumes)
view = views.InstancesView(servers, req=req)
paged = pagination.SimplePaginatedDataView(req.url, 'instances', view,
marker)
return wsgi.Result(paged.data(), 200)
@ -151,9 +150,8 @@ class InstanceController(wsgi.Controller):
context = req.environ[wsgi.CONTEXT_KEY]
server = models.load_instance_with_guest(models.DetailInstance,
context, id)
return wsgi.Result(views.InstanceDetailView(server, req=req,
add_addresses=self.add_addresses,
add_volumes=self.add_volumes).data(), 200)
return wsgi.Result(views.InstanceDetailView(server,
req=req).data(), 200)
def delete(self, req, tenant_id, id):
"""Delete a single instance."""
@ -169,25 +167,13 @@ class InstanceController(wsgi.Controller):
@api_validation(action="create")
def create(self, req, body, tenant_id):
# find the service id (cant be done yet at startup due to
# inconsistencies w/ the load app paste
# TODO(hub-cap): figure out how to get this to work in __init__ time
# TODO(hub-cap): The problem with this in __init__ is that the paste
# config is generated w/ the same config file as the db flags that
# are needed for init. These need to be split so the db can be init'd
# w/o the paste stuff. Since the paste stuff inits the
# database.service module, it is a chicken before the egg problem.
# Simple refactor will fix it and we can move this into the __init__
# code. Or maybe we shouldnt due to the nature of changing images.
# This needs discussion.
# TODO(hub-cap): turn this into middleware
LOG.info(_("Creating a database instance for tenant '%s'") % tenant_id)
LOG.info(_("req : '%s'\n\n") % req)
LOG.info(_("body : '%s'\n\n") % body)
context = req.environ[wsgi.CONTEXT_KEY]
service_type = body['instance'].get('service_type')
if service_type is None:
service_type = 'mysql'
# Set the service type to mysql if its not in the request
service_type = body['instance'].get('service_type') or 'mysql'
service = models.ServiceImage.find_by(service_name=service_type)
image_id = service['image_id']
name = body['instance']['name']
@ -217,8 +203,7 @@ class InstanceController(wsgi.Controller):
image_id, databases, users,
service_type, volume_size)
view = views.InstanceDetailView(instance, req=req,
add_volumes=self.add_volumes)
view = views.InstanceDetailView(instance, req=req)
return wsgi.Result(view.data(), 200)
@staticmethod
@ -271,8 +256,7 @@ class InstanceController(wsgi.Controller):
name = body['instance'].get('name', '').strip()
if not name:
raise exception.MissingKey(key='name')
vol_enabled = CONF.reddwarf_volume_support
if vol_enabled:
if CONF.reddwarf_volume_support:
if body['instance'].get('volume', None):
if body['instance']['volume'].get('size', None):
volume_size = body['instance']['volume']['size']

27
reddwarf/instance/views.py
View File

@ -40,11 +40,8 @@ def get_ip_address(addresses):
class InstanceView(object):
"""Uses a SimpleInstance."""
def __init__(self, instance, req=None, add_addresses=False,
add_volumes=False):
def __init__(self, instance, req=None):
self.instance = instance
self.add_addresses = add_addresses
self.add_volumes = add_volumes
self.req = req
def data(self):
@ -55,7 +52,7 @@ class InstanceView(object):
"links": self._build_links(),
"flavor": self._build_flavor_info(),
}
if self.add_volumes:
if CONF.reddwarf_volume_support:
instance_dict['volume'] = {'size': self.instance.volume_size}
LOG.debug(instance_dict)
return {"instance": instance_dict}
@ -77,13 +74,9 @@ class InstanceView(object):
class InstanceDetailView(InstanceView):
"""Works with a full-blown instance."""
def __init__(self, instance, req, add_addresses=False,
add_volumes=False):
def __init__(self, instance, req):
super(InstanceDetailView, self).__init__(instance,
req=req,
add_volumes=add_volumes)
self.add_addresses = add_addresses
self.add_volumes = add_volumes
req=req)
def _to_gb(self, bytes):
return bytes / 1024.0 ** 3
@ -97,11 +90,11 @@ class InstanceDetailView(InstanceView):
if dns_support:
result['instance']['hostname'] = self.instance.hostname
if self.add_addresses:
if CONF.add_addresses:
ip = get_ip_address(self.instance.addresses)
if ip is not None and len(ip) > 0:
result['instance']['ip'] = ip
if self.add_volumes:
if CONF.reddwarf_volume_support:
if (isinstance(self.instance, models.DetailInstance) and
self.instance.volume_used):
used = self._to_gb(self.instance.volume_used)
@ -112,12 +105,9 @@ class InstanceDetailView(InstanceView):
class InstancesView(object):
"""Shows a list of SimpleInstance objects."""
def __init__(self, instances, req=None, add_addresses=False,
add_volumes=True):
def __init__(self, instances, req=None):
self.instances = instances
self.req = req
self.add_addresses = add_addresses
self.add_volumes = add_volumes
def data(self):
data = []
@ -127,6 +117,5 @@ class InstancesView(object):
return {'instances': data}
def data_for_instance(self, instance):
view = InstanceView(instance, req=self.req,
add_volumes=self.add_volumes)
view = InstanceView(instance, req=self.req)
return view.data()['instance']

4
reddwarf/taskmanager/api.py
View File

@ -31,6 +31,10 @@ CONF = cfg.CONF
LOG = logging.getLogger(__name__)
#todo(hub-cap): find a better way to deal w/ the fakes. Im sure we can
# use a fake impl to deal w/ this and switch it out in the configs.
# The ManagerAPI is only used here and should eventually be removed when
# we have a better way to handle fake casts (see rpc fake_impl)
class API(ManagerAPI):
"""API for interacting with the task manager."""

1
reddwarf/taskmanager/manager.py
View File

@ -20,7 +20,6 @@ import traceback
from eventlet import greenthread
from reddwarf.common import exception
from reddwarf.common import service
from reddwarf.openstack.common import log as logging
from reddwarf.openstack.common import periodic_task
from reddwarf.openstack.common.rpc.common import UnsupportedRpcVersion