Clean-up openstack.common
* Unused module was removed * Existing modules were updated * Readme file was added * Exceptions classes, used from unsupported exceptions from oslo-incubator are moved to murano.common * Orginize imports to the correct order in files, where the order were modified Next patch will remove all locations of openstack.common.log usage Change-Id: I4382215df1bcf81aea60e29039de548bcfe5a356
This commit is contained in:
parent
54b40adb40
commit
cb2d5d0caa
|
@ -24,11 +24,11 @@ from webob import exc
|
||||||
|
|
||||||
import murano.api.v1
|
import murano.api.v1
|
||||||
from murano.api.v1 import schemas
|
from murano.api.v1 import schemas
|
||||||
|
from murano.common import exceptions
|
||||||
from murano.common import policy
|
from murano.common import policy
|
||||||
from murano.common import wsgi
|
from murano.common import wsgi
|
||||||
from murano.db.catalog import api as db_api
|
from murano.db.catalog import api as db_api
|
||||||
from murano.common.i18n import _, _LW
|
from murano.common.i18n import _, _LW
|
||||||
from murano.openstack.common import exception
|
|
||||||
from murano.openstack.common import log as logging
|
from murano.openstack.common import log as logging
|
||||||
from murano.packages import exceptions as pkg_exc
|
from murano.packages import exceptions as pkg_exc
|
||||||
from murano.packages import load_utils
|
from murano.packages import load_utils
|
||||||
|
@ -46,7 +46,7 @@ PKG_PARAMS_MAP = murano.api.v1.PKG_PARAMS_MAP
|
||||||
def _check_content_type(req, content_type):
|
def _check_content_type(req, content_type):
|
||||||
try:
|
try:
|
||||||
req.get_content_type((content_type,))
|
req.get_content_type((content_type,))
|
||||||
except exception.InvalidContentType:
|
except exceptions.InvalidContentType:
|
||||||
msg = _("Content-Type must be '{0}'").format(content_type)
|
msg = _("Content-Type must be '{0}'").format(content_type)
|
||||||
LOG.error(msg)
|
LOG.error(msg)
|
||||||
raise exc.HTTPBadRequest(explanation=msg)
|
raise exc.HTTPBadRequest(explanation=msg)
|
||||||
|
|
|
@ -12,6 +12,49 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
_FATAL_EXCEPTION_FORMAT_ERRORS = False
|
||||||
|
|
||||||
|
# Exceptions from openstack-common
|
||||||
|
|
||||||
|
|
||||||
|
class Error(Exception):
|
||||||
|
def __init__(self, message=None):
|
||||||
|
super(Error, self).__init__(message)
|
||||||
|
|
||||||
|
|
||||||
|
class OpenstackException(Exception):
|
||||||
|
"""Base Exception class.
|
||||||
|
|
||||||
|
To correctly use this class, inherit from it and define
|
||||||
|
a 'msg_fmt' property. That message will get printf'd
|
||||||
|
with the keyword arguments provided to the constructor.
|
||||||
|
"""
|
||||||
|
msg_fmt = "An unknown exception occurred"
|
||||||
|
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
try:
|
||||||
|
self._error_string = self.msg_fmt % kwargs
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
if _FATAL_EXCEPTION_FORMAT_ERRORS:
|
||||||
|
raise
|
||||||
|
else:
|
||||||
|
# at least get the io.murano message out if something happened
|
||||||
|
self._error_string = self.msg_fmt
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return self._error_string
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidContentType(OpenstackException):
|
||||||
|
msg_fmt = "Invalid content type %(content_type)s"
|
||||||
|
|
||||||
|
|
||||||
|
class MalformedRequestBody(OpenstackException):
|
||||||
|
msg_fmt = "Malformed message body: %(reason)s"
|
||||||
|
|
||||||
|
# Murano exceptions
|
||||||
|
|
||||||
|
|
||||||
class TimeoutException(Exception):
|
class TimeoutException(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
|
@ -36,9 +36,9 @@ import webob.dec
|
||||||
import webob.exc
|
import webob.exc
|
||||||
|
|
||||||
from murano.api.v1 import schemas
|
from murano.api.v1 import schemas
|
||||||
|
from murano.common import exceptions
|
||||||
from murano.common.i18n import _
|
from murano.common.i18n import _
|
||||||
from murano.common import xmlutils
|
from murano.common import xmlutils
|
||||||
from murano.openstack.common import exception
|
|
||||||
from murano.openstack.common import log as logging
|
from murano.openstack.common import log as logging
|
||||||
from murano.openstack.common import service
|
from murano.openstack.common import service
|
||||||
from murano.openstack.common import sslutils
|
from murano.openstack.common import sslutils
|
||||||
|
@ -321,7 +321,7 @@ class Request(webob.Request):
|
||||||
self.default_request_content_types)
|
self.default_request_content_types)
|
||||||
|
|
||||||
if content_type not in allowed_content_types:
|
if content_type not in allowed_content_types:
|
||||||
raise exception.InvalidContentType(content_type=content_type)
|
raise exceptions.InvalidContentType(content_type=content_type)
|
||||||
return content_type
|
return content_type
|
||||||
|
|
||||||
|
|
||||||
|
@ -360,10 +360,10 @@ class Resource(object):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
action, action_args, accept = self.deserialize_request(request)
|
action, action_args, accept = self.deserialize_request(request)
|
||||||
except exception.InvalidContentType:
|
except exceptions.InvalidContentType:
|
||||||
msg = _("Unsupported Content-Type")
|
msg = _("Unsupported Content-Type")
|
||||||
return webob.exc.HTTPUnsupportedMediaType(explanation=msg)
|
return webob.exc.HTTPUnsupportedMediaType(explanation=msg)
|
||||||
except exception.MalformedRequestBody:
|
except exceptions.MalformedRequestBody:
|
||||||
msg = _("Malformed request body")
|
msg = _("Malformed request body")
|
||||||
return webob.exc.HTTPBadRequest(explanation=msg)
|
return webob.exc.HTTPBadRequest(explanation=msg)
|
||||||
|
|
||||||
|
@ -604,7 +604,7 @@ class ResponseSerializer(object):
|
||||||
try:
|
try:
|
||||||
return self.body_serializers[content_type]
|
return self.body_serializers[content_type]
|
||||||
except (KeyError, TypeError):
|
except (KeyError, TypeError):
|
||||||
raise exception.InvalidContentType(content_type=content_type)
|
raise exceptions.InvalidContentType(content_type=content_type)
|
||||||
|
|
||||||
|
|
||||||
class RequestHeadersDeserializer(ActionDispatcher):
|
class RequestHeadersDeserializer(ActionDispatcher):
|
||||||
|
@ -665,7 +665,7 @@ class RequestDeserializer(object):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
content_type = request.get_content_type()
|
content_type = request.get_content_type()
|
||||||
except exception.InvalidContentType as e:
|
except exceptions.InvalidContentType as e:
|
||||||
msg = "Unrecognized Content-Type provided in request: {0}"
|
msg = "Unrecognized Content-Type provided in request: {0}"
|
||||||
LOG.debug(unicode(msg).format(str(e)))
|
LOG.debug(unicode(msg).format(str(e)))
|
||||||
raise
|
raise
|
||||||
|
@ -676,7 +676,7 @@ class RequestDeserializer(object):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
deserializer = self.get_body_deserializer(content_type)
|
deserializer = self.get_body_deserializer(content_type)
|
||||||
except exception.InvalidContentType:
|
except exceptions.InvalidContentType:
|
||||||
LOG.debug("Unable to deserialize body as provided Content-Type")
|
LOG.debug("Unable to deserialize body as provided Content-Type")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
@ -686,7 +686,7 @@ class RequestDeserializer(object):
|
||||||
try:
|
try:
|
||||||
return self.body_deserializers[content_type]
|
return self.body_deserializers[content_type]
|
||||||
except (KeyError, TypeError):
|
except (KeyError, TypeError):
|
||||||
raise exception.InvalidContentType(content_type=content_type)
|
raise exceptions.InvalidContentType(content_type=content_type)
|
||||||
|
|
||||||
def get_expected_content_type(self, request):
|
def get_expected_content_type(self, request):
|
||||||
return request.best_match_content_type(self.supported_content_types)
|
return request.best_match_content_type(self.supported_content_types)
|
||||||
|
@ -727,7 +727,7 @@ class JSONDeserializer(TextDeserializer):
|
||||||
return jsonutils.loads(datastring)
|
return jsonutils.loads(datastring)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
msg = _("cannot understand JSON")
|
msg = _("cannot understand JSON")
|
||||||
raise exception.MalformedRequestBody(reason=msg)
|
raise exceptions.MalformedRequestBody(reason=msg)
|
||||||
|
|
||||||
def default(self, request):
|
def default(self, request):
|
||||||
datastring = request.body
|
datastring = request.body
|
||||||
|
@ -747,7 +747,7 @@ class JSONPatchDeserializer(TextDeserializer):
|
||||||
operations = jsonutils.loads(datastring)
|
operations = jsonutils.loads(datastring)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
msg = _("cannot understand JSON")
|
msg = _("cannot understand JSON")
|
||||||
raise exception.MalformedRequestBody(reason=msg)
|
raise exceptions.MalformedRequestBody(reason=msg)
|
||||||
|
|
||||||
changes = []
|
changes = []
|
||||||
for raw_change in operations:
|
for raw_change in operations:
|
||||||
|
@ -886,7 +886,7 @@ class XMLDeserializer(TextDeserializer):
|
||||||
return {node.nodeName: self._from_xml_node(node, plurals)}
|
return {node.nodeName: self._from_xml_node(node, plurals)}
|
||||||
except expat.ExpatError:
|
except expat.ExpatError:
|
||||||
msg = _("cannot understand XML")
|
msg = _("cannot understand XML")
|
||||||
raise exception.MalformedRequestBody(reason=msg)
|
raise exceptions.MalformedRequestBody(reason=msg)
|
||||||
|
|
||||||
def _from_xml_node(self, node, listnames):
|
def _from_xml_node(self, node, listnames):
|
||||||
"""Convert a minidom node to a simple Python type.
|
"""Convert a minidom node to a simple Python type.
|
||||||
|
|
|
@ -0,0 +1,16 @@
|
||||||
|
oslo-incubator
|
||||||
|
--------------
|
||||||
|
|
||||||
|
A number of modules from oslo-incubator are imported into this project.
|
||||||
|
You can clone the oslo-incubator repository using the following url:
|
||||||
|
|
||||||
|
git://git.openstack.org/openstack/oslo-incubator
|
||||||
|
|
||||||
|
These modules are "incubating" in oslo-incubator and are kept in sync
|
||||||
|
with the help of oslo-incubator's update.py script. See:
|
||||||
|
|
||||||
|
https://wiki.openstack.org/wiki/Oslo#Syncing_Code_from_Incubator
|
||||||
|
|
||||||
|
The copy of the code should never be directly modified here. Please
|
||||||
|
always update oslo-incubator first and then run the script to copy
|
||||||
|
the changes across.
|
|
@ -19,19 +19,18 @@ from __future__ import print_function
|
||||||
import copy
|
import copy
|
||||||
import errno
|
import errno
|
||||||
import gc
|
import gc
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import pprint
|
import pprint
|
||||||
import socket
|
import socket
|
||||||
import sys
|
import sys
|
||||||
import traceback
|
import traceback
|
||||||
|
|
||||||
import eventlet
|
|
||||||
import eventlet.backdoor
|
import eventlet.backdoor
|
||||||
import greenlet
|
import greenlet
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
|
|
||||||
from murano.openstack.common._i18n import _LI
|
from murano.openstack.common._i18n import _LI
|
||||||
from murano.openstack.common import log as logging
|
|
||||||
|
|
||||||
help_for_backdoor_port = (
|
help_for_backdoor_port = (
|
||||||
"Acceptable values are 0, <port>, and <start>:<end>, where 0 results "
|
"Acceptable values are 0, <port>, and <start>:<end>, where 0 results "
|
||||||
|
@ -51,7 +50,7 @@ LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def list_opts():
|
def list_opts():
|
||||||
"""Entry point for oslo.config-generator.
|
"""Entry point for oslo-config-generator.
|
||||||
"""
|
"""
|
||||||
return [(None, copy.deepcopy(eventlet_backdoor_opts))]
|
return [(None, copy.deepcopy(eventlet_backdoor_opts))]
|
||||||
|
|
||||||
|
@ -144,7 +143,7 @@ def initialize_if_enabled():
|
||||||
# listen(). In any case, pull the port number out here.
|
# listen(). In any case, pull the port number out here.
|
||||||
port = sock.getsockname()[1]
|
port = sock.getsockname()[1]
|
||||||
LOG.info(
|
LOG.info(
|
||||||
_LI('Eventlet backdoor listening on %(port)s for process %(pid)d') %
|
_LI('Eventlet backdoor listening on %(port)s for process %(pid)d'),
|
||||||
{'port': port, 'pid': os.getpid()}
|
{'port': port, 'pid': os.getpid()}
|
||||||
)
|
)
|
||||||
eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock,
|
eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock,
|
||||||
|
|
|
@ -1,139 +0,0 @@
|
||||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
|
||||||
|
|
||||||
# Copyright 2011 OpenStack Foundation.
|
|
||||||
# All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
"""
|
|
||||||
Exceptions common to OpenStack projects
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
|
|
||||||
from murano.openstack.common._i18n import _
|
|
||||||
|
|
||||||
_FATAL_EXCEPTION_FORMAT_ERRORS = False
|
|
||||||
|
|
||||||
|
|
||||||
class Error(Exception):
|
|
||||||
def __init__(self, message=None):
|
|
||||||
super(Error, self).__init__(message)
|
|
||||||
|
|
||||||
|
|
||||||
class ApiError(Error):
|
|
||||||
def __init__(self, message='Unknown', code='Unknown'):
|
|
||||||
self.api_message = message
|
|
||||||
self.code = code
|
|
||||||
super(ApiError, self).__init__('%s: %s' % (code, message))
|
|
||||||
|
|
||||||
|
|
||||||
class NotFound(Error):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class UnknownScheme(Error):
|
|
||||||
|
|
||||||
msg_fmt = "Unknown scheme '%s' found in URI"
|
|
||||||
|
|
||||||
def __init__(self, scheme):
|
|
||||||
msg = self.msg_fmt % scheme
|
|
||||||
super(UnknownScheme, self).__init__(msg)
|
|
||||||
|
|
||||||
|
|
||||||
class BadStoreUri(Error):
|
|
||||||
|
|
||||||
msg_fmt = "The Store URI %s was malformed. Reason: %s"
|
|
||||||
|
|
||||||
def __init__(self, uri, reason):
|
|
||||||
msg = self.msg_fmt % (uri, reason)
|
|
||||||
super(BadStoreUri, self).__init__(msg)
|
|
||||||
|
|
||||||
|
|
||||||
class Duplicate(Error):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class NotAuthorized(Error):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class NotEmpty(Error):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class Invalid(Error):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class BadInputError(Exception):
|
|
||||||
"""Error resulting from a client sending bad input to a server"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class MissingArgumentError(Error):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class DatabaseMigrationError(Error):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class ClientConnectionError(Exception):
|
|
||||||
"""Error resulting from a client connecting to a server"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def wrap_exception(f):
|
|
||||||
def _wrap(*args, **kw):
|
|
||||||
try:
|
|
||||||
return f(*args, **kw)
|
|
||||||
except Exception as e:
|
|
||||||
if not isinstance(e, Error):
|
|
||||||
logging.exception(_('Uncaught exception'))
|
|
||||||
raise Error(str(e))
|
|
||||||
raise
|
|
||||||
_wrap.func_name = f.func_name
|
|
||||||
return _wrap
|
|
||||||
|
|
||||||
|
|
||||||
class OpenstackException(Exception):
|
|
||||||
"""Base Exception class.
|
|
||||||
|
|
||||||
To correctly use this class, inherit from it and define
|
|
||||||
a 'msg_fmt' property. That message will get printf'd
|
|
||||||
with the keyword arguments provided to the constructor.
|
|
||||||
"""
|
|
||||||
msg_fmt = "An unknown exception occurred"
|
|
||||||
|
|
||||||
def __init__(self, **kwargs):
|
|
||||||
try:
|
|
||||||
self._error_string = self.msg_fmt % kwargs
|
|
||||||
|
|
||||||
except Exception:
|
|
||||||
if _FATAL_EXCEPTION_FORMAT_ERRORS:
|
|
||||||
raise
|
|
||||||
else:
|
|
||||||
# at least get the io.murano message out if something happened
|
|
||||||
self._error_string = self.msg_fmt
|
|
||||||
|
|
||||||
def __str__(self):
|
|
||||||
return self._error_string
|
|
||||||
|
|
||||||
|
|
||||||
class MalformedRequestBody(OpenstackException):
|
|
||||||
msg_fmt = "Malformed message body: %(reason)s"
|
|
||||||
|
|
||||||
|
|
||||||
class InvalidContentType(OpenstackException):
|
|
||||||
msg_fmt = "Invalid content type %(content_type)s"
|
|
|
@ -1,146 +0,0 @@
|
||||||
# Copyright 2011 OpenStack Foundation.
|
|
||||||
# All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
import contextlib
|
|
||||||
import errno
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
from oslo_utils import excutils
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_FILE_CACHE = {}
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_tree(path):
|
|
||||||
"""Create a directory (and any ancestor directories required)
|
|
||||||
|
|
||||||
:param path: Directory to create
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
os.makedirs(path)
|
|
||||||
except OSError as exc:
|
|
||||||
if exc.errno == errno.EEXIST:
|
|
||||||
if not os.path.isdir(path):
|
|
||||||
raise
|
|
||||||
else:
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
def read_cached_file(filename, force_reload=False):
|
|
||||||
"""Read from a file if it has been modified.
|
|
||||||
|
|
||||||
:param force_reload: Whether to reload the file.
|
|
||||||
:returns: A tuple with a boolean specifying if the data is fresh
|
|
||||||
or not.
|
|
||||||
"""
|
|
||||||
global _FILE_CACHE
|
|
||||||
|
|
||||||
if force_reload:
|
|
||||||
delete_cached_file(filename)
|
|
||||||
|
|
||||||
reloaded = False
|
|
||||||
mtime = os.path.getmtime(filename)
|
|
||||||
cache_info = _FILE_CACHE.setdefault(filename, {})
|
|
||||||
|
|
||||||
if not cache_info or mtime > cache_info.get('mtime', 0):
|
|
||||||
LOG.debug("Reloading cached file %s" % filename)
|
|
||||||
with open(filename) as fap:
|
|
||||||
cache_info['data'] = fap.read()
|
|
||||||
cache_info['mtime'] = mtime
|
|
||||||
reloaded = True
|
|
||||||
return (reloaded, cache_info['data'])
|
|
||||||
|
|
||||||
|
|
||||||
def delete_cached_file(filename):
|
|
||||||
"""Delete cached file if present.
|
|
||||||
|
|
||||||
:param filename: filename to delete
|
|
||||||
"""
|
|
||||||
global _FILE_CACHE
|
|
||||||
|
|
||||||
if filename in _FILE_CACHE:
|
|
||||||
del _FILE_CACHE[filename]
|
|
||||||
|
|
||||||
|
|
||||||
def delete_if_exists(path, remove=os.unlink):
|
|
||||||
"""Delete a file, but ignore file not found error.
|
|
||||||
|
|
||||||
:param path: File to delete
|
|
||||||
:param remove: Optional function to remove passed path
|
|
||||||
"""
|
|
||||||
|
|
||||||
try:
|
|
||||||
remove(path)
|
|
||||||
except OSError as e:
|
|
||||||
if e.errno != errno.ENOENT:
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
@contextlib.contextmanager
|
|
||||||
def remove_path_on_error(path, remove=delete_if_exists):
|
|
||||||
"""Protect code that wants to operate on PATH atomically.
|
|
||||||
Any exception will cause PATH to be removed.
|
|
||||||
|
|
||||||
:param path: File to work with
|
|
||||||
:param remove: Optional function to remove passed path
|
|
||||||
"""
|
|
||||||
|
|
||||||
try:
|
|
||||||
yield
|
|
||||||
except Exception:
|
|
||||||
with excutils.save_and_reraise_exception():
|
|
||||||
remove(path)
|
|
||||||
|
|
||||||
|
|
||||||
def file_open(*args, **kwargs):
|
|
||||||
"""Open file
|
|
||||||
|
|
||||||
see built-in open() documentation for more details
|
|
||||||
|
|
||||||
Note: The reason this is kept in a separate module is to easily
|
|
||||||
be able to provide a stub module that doesn't alter system
|
|
||||||
state at all (for unit tests)
|
|
||||||
"""
|
|
||||||
return open(*args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def write_to_tempfile(content, path=None, suffix='', prefix='tmp'):
|
|
||||||
"""Create temporary file or use existing file.
|
|
||||||
|
|
||||||
This util is needed for creating temporary file with
|
|
||||||
specified content, suffix and prefix. If path is not None,
|
|
||||||
it will be used for writing content. If the path doesn't
|
|
||||||
exist it'll be created.
|
|
||||||
|
|
||||||
:param content: content for temporary file.
|
|
||||||
:param path: same as parameter 'dir' for mkstemp
|
|
||||||
:param suffix: same as parameter 'suffix' for mkstemp
|
|
||||||
:param prefix: same as parameter 'prefix' for mkstemp
|
|
||||||
|
|
||||||
For example: it can be used in database tests for creating
|
|
||||||
configuration files.
|
|
||||||
"""
|
|
||||||
if path:
|
|
||||||
ensure_tree(path)
|
|
||||||
|
|
||||||
(fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix)
|
|
||||||
try:
|
|
||||||
os.write(fd, content)
|
|
||||||
finally:
|
|
||||||
os.close(fd)
|
|
||||||
return path
|
|
|
@ -1,326 +0,0 @@
|
||||||
# Copyright 2011 OpenStack Foundation.
|
|
||||||
# All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
import contextlib
|
|
||||||
import errno
|
|
||||||
import functools
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
import threading
|
|
||||||
import time
|
|
||||||
import weakref
|
|
||||||
|
|
||||||
from oslo_config import cfg
|
|
||||||
|
|
||||||
from murano.openstack.common import fileutils
|
|
||||||
from murano.openstack.common._i18n import _, _LE, _LI
|
|
||||||
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
util_opts = [
|
|
||||||
cfg.BoolOpt('disable_process_locking', default=False,
|
|
||||||
help='Enables or disables inter-process locks.'),
|
|
||||||
cfg.StrOpt('lock_path',
|
|
||||||
default=os.environ.get("MURANO_LOCK_PATH"),
|
|
||||||
help='Directory to use for lock files.')
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
CONF = cfg.CONF
|
|
||||||
CONF.register_opts(util_opts)
|
|
||||||
|
|
||||||
|
|
||||||
def set_defaults(lock_path):
|
|
||||||
cfg.set_defaults(util_opts, lock_path=lock_path)
|
|
||||||
|
|
||||||
|
|
||||||
class _FileLock(object):
|
|
||||||
"""Lock implementation which allows multiple locks, working around
|
|
||||||
issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does
|
|
||||||
not require any cleanup. Since the lock is always held on a file
|
|
||||||
descriptor rather than outside of the process, the lock gets dropped
|
|
||||||
automatically if the process crashes, even if __exit__ is not executed.
|
|
||||||
|
|
||||||
There are no guarantees regarding usage by multiple green threads in a
|
|
||||||
single process here. This lock works only between processes. Exclusive
|
|
||||||
access between local threads should be achieved using the semaphores
|
|
||||||
in the @synchronized decorator.
|
|
||||||
|
|
||||||
Note these locks are released when the descriptor is closed, so it's not
|
|
||||||
safe to close the file descriptor while another green thread holds the
|
|
||||||
lock. Just opening and closing the lock file can break synchronisation,
|
|
||||||
so lock files must be accessed only using this abstraction.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, name):
|
|
||||||
self.lockfile = None
|
|
||||||
self.fname = name
|
|
||||||
|
|
||||||
def acquire(self):
|
|
||||||
basedir = os.path.dirname(self.fname)
|
|
||||||
|
|
||||||
if not os.path.exists(basedir):
|
|
||||||
fileutils.ensure_tree(basedir)
|
|
||||||
LOG.info(_LI('Created lock path: %s'), basedir)
|
|
||||||
|
|
||||||
self.lockfile = open(self.fname, 'w')
|
|
||||||
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
# Using non-blocking locks since green threads are not
|
|
||||||
# patched to deal with blocking locking calls.
|
|
||||||
# Also upon reading the MSDN docs for locking(), it seems
|
|
||||||
# to have a laughable 10 attempts "blocking" mechanism.
|
|
||||||
self.trylock()
|
|
||||||
LOG.debug('Got file lock "%s"', self.fname)
|
|
||||||
return True
|
|
||||||
except IOError as e:
|
|
||||||
if e.errno in (errno.EACCES, errno.EAGAIN):
|
|
||||||
# external locks synchronise things like iptables
|
|
||||||
# updates - give it some time to prevent busy spinning
|
|
||||||
time.sleep(0.01)
|
|
||||||
else:
|
|
||||||
raise threading.ThreadError(_("Unable to acquire lock on"
|
|
||||||
" `%(filename)s` due to"
|
|
||||||
" %(exception)s") %
|
|
||||||
{'filename': self.fname,
|
|
||||||
'exception': e})
|
|
||||||
|
|
||||||
def __enter__(self):
|
|
||||||
self.acquire()
|
|
||||||
return self
|
|
||||||
|
|
||||||
def release(self):
|
|
||||||
try:
|
|
||||||
self.unlock()
|
|
||||||
self.lockfile.close()
|
|
||||||
LOG.debug('Released file lock "%s"', self.fname)
|
|
||||||
except IOError:
|
|
||||||
LOG.exception(_LE("Could not release the acquired lock `%s`"),
|
|
||||||
self.fname)
|
|
||||||
|
|
||||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
||||||
self.release()
|
|
||||||
|
|
||||||
def exists(self):
|
|
||||||
return os.path.exists(self.fname)
|
|
||||||
|
|
||||||
def trylock(self):
|
|
||||||
raise NotImplementedError()
|
|
||||||
|
|
||||||
def unlock(self):
|
|
||||||
raise NotImplementedError()
|
|
||||||
|
|
||||||
|
|
||||||
class _WindowsLock(_FileLock):
|
|
||||||
def trylock(self):
|
|
||||||
msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1)
|
|
||||||
|
|
||||||
def unlock(self):
|
|
||||||
msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1)
|
|
||||||
|
|
||||||
|
|
||||||
class _FcntlLock(_FileLock):
|
|
||||||
def trylock(self):
|
|
||||||
fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
||||||
|
|
||||||
def unlock(self):
|
|
||||||
fcntl.lockf(self.lockfile, fcntl.LOCK_UN)
|
|
||||||
|
|
||||||
|
|
||||||
if os.name == 'nt':
|
|
||||||
import msvcrt
|
|
||||||
InterProcessLock = _WindowsLock
|
|
||||||
else:
|
|
||||||
import fcntl
|
|
||||||
InterProcessLock = _FcntlLock
|
|
||||||
|
|
||||||
_semaphores = weakref.WeakValueDictionary()
|
|
||||||
_semaphores_lock = threading.Lock()
|
|
||||||
|
|
||||||
|
|
||||||
def _get_lock_path(name, lock_file_prefix, lock_path=None):
|
|
||||||
# NOTE(mikal): the lock name cannot contain directory
|
|
||||||
# separators
|
|
||||||
name = name.replace(os.sep, '_')
|
|
||||||
if lock_file_prefix:
|
|
||||||
sep = '' if lock_file_prefix.endswith('-') else '-'
|
|
||||||
name = '%s%s%s' % (lock_file_prefix, sep, name)
|
|
||||||
|
|
||||||
local_lock_path = lock_path or CONF.lock_path
|
|
||||||
|
|
||||||
if not local_lock_path:
|
|
||||||
raise cfg.RequiredOptError('lock_path')
|
|
||||||
|
|
||||||
return os.path.join(local_lock_path, name)
|
|
||||||
|
|
||||||
|
|
||||||
def external_lock(name, lock_file_prefix=None, lock_path=None):
|
|
||||||
LOG.debug('Attempting to grab external lock "%(lock)s"',
|
|
||||||
{'lock': name})
|
|
||||||
|
|
||||||
lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path)
|
|
||||||
|
|
||||||
return InterProcessLock(lock_file_path)
|
|
||||||
|
|
||||||
|
|
||||||
def remove_external_lock_file(name, lock_file_prefix=None):
|
|
||||||
"""Remove an external lock file when it's not used anymore
|
|
||||||
This will be helpful when we have a lot of lock files
|
|
||||||
"""
|
|
||||||
with internal_lock(name):
|
|
||||||
lock_file_path = _get_lock_path(name, lock_file_prefix)
|
|
||||||
try:
|
|
||||||
os.remove(lock_file_path)
|
|
||||||
except OSError:
|
|
||||||
LOG.info(_LI('Failed to remove file %(file)s'),
|
|
||||||
{'file': lock_file_path})
|
|
||||||
|
|
||||||
|
|
||||||
def internal_lock(name):
|
|
||||||
with _semaphores_lock:
|
|
||||||
try:
|
|
||||||
sem = _semaphores[name]
|
|
||||||
LOG.debug('Using existing semaphore "%s"', name)
|
|
||||||
except KeyError:
|
|
||||||
sem = threading.Semaphore()
|
|
||||||
_semaphores[name] = sem
|
|
||||||
LOG.debug('Created new semaphore "%s"', name)
|
|
||||||
|
|
||||||
return sem
|
|
||||||
|
|
||||||
|
|
||||||
@contextlib.contextmanager
|
|
||||||
def lock(name, lock_file_prefix=None, external=False, lock_path=None):
|
|
||||||
"""Context based lock
|
|
||||||
|
|
||||||
This function yields a `threading.Semaphore` instance (if we don't use
|
|
||||||
eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is
|
|
||||||
True, in which case, it'll yield an InterProcessLock instance.
|
|
||||||
|
|
||||||
:param lock_file_prefix: The lock_file_prefix argument is used to provide
|
|
||||||
lock files on disk with a meaningful prefix.
|
|
||||||
|
|
||||||
:param external: The external keyword argument denotes whether this lock
|
|
||||||
should work across multiple processes. This means that if two different
|
|
||||||
workers both run a method decorated with @synchronized('mylock',
|
|
||||||
external=True), only one of them will execute at a time.
|
|
||||||
"""
|
|
||||||
int_lock = internal_lock(name)
|
|
||||||
with int_lock:
|
|
||||||
LOG.debug('Acquired semaphore "%(lock)s"', {'lock': name})
|
|
||||||
try:
|
|
||||||
if external and not CONF.disable_process_locking:
|
|
||||||
ext_lock = external_lock(name, lock_file_prefix, lock_path)
|
|
||||||
with ext_lock:
|
|
||||||
yield ext_lock
|
|
||||||
else:
|
|
||||||
yield int_lock
|
|
||||||
finally:
|
|
||||||
LOG.debug('Releasing semaphore "%(lock)s"', {'lock': name})
|
|
||||||
|
|
||||||
|
|
||||||
def synchronized(name, lock_file_prefix=None, external=False, lock_path=None):
|
|
||||||
"""Synchronization decorator.
|
|
||||||
|
|
||||||
Decorating a method like so::
|
|
||||||
|
|
||||||
@synchronized('mylock')
|
|
||||||
def foo(self, *args):
|
|
||||||
...
|
|
||||||
|
|
||||||
ensures that only one thread will execute the foo method at a time.
|
|
||||||
|
|
||||||
Different methods can share the same lock::
|
|
||||||
|
|
||||||
@synchronized('mylock')
|
|
||||||
def foo(self, *args):
|
|
||||||
...
|
|
||||||
|
|
||||||
@synchronized('mylock')
|
|
||||||
def bar(self, *args):
|
|
||||||
...
|
|
||||||
|
|
||||||
This way only one of either foo or bar can be executing at a time.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def wrap(f):
|
|
||||||
@functools.wraps(f)
|
|
||||||
def inner(*args, **kwargs):
|
|
||||||
try:
|
|
||||||
with lock(name, lock_file_prefix, external, lock_path):
|
|
||||||
LOG.debug('Got semaphore / lock "%(function)s"',
|
|
||||||
{'function': f.__name__})
|
|
||||||
return f(*args, **kwargs)
|
|
||||||
finally:
|
|
||||||
LOG.debug('Semaphore / lock released "%(function)s"',
|
|
||||||
{'function': f.__name__})
|
|
||||||
return inner
|
|
||||||
return wrap
|
|
||||||
|
|
||||||
|
|
||||||
def synchronized_with_prefix(lock_file_prefix):
|
|
||||||
"""Partial object generator for the synchronization decorator.
|
|
||||||
|
|
||||||
Redefine @synchronized in each project like so::
|
|
||||||
|
|
||||||
(in nova/utils.py)
|
|
||||||
from nova.openstack.common import lockutils
|
|
||||||
|
|
||||||
synchronized = lockutils.synchronized_with_prefix('nova-')
|
|
||||||
|
|
||||||
|
|
||||||
(in nova/foo.py)
|
|
||||||
from nova import utils
|
|
||||||
|
|
||||||
@utils.synchronized('mylock')
|
|
||||||
def bar(self, *args):
|
|
||||||
...
|
|
||||||
|
|
||||||
The lock_file_prefix argument is used to provide lock files on disk with a
|
|
||||||
meaningful prefix.
|
|
||||||
"""
|
|
||||||
|
|
||||||
return functools.partial(synchronized, lock_file_prefix=lock_file_prefix)
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv):
|
|
||||||
"""Create a dir for locks and pass it to command from arguments
|
|
||||||
|
|
||||||
If you run this:
|
|
||||||
python -m openstack.common.lockutils python setup.py testr <etc>
|
|
||||||
|
|
||||||
a temporary directory will be created for all your locks and passed to all
|
|
||||||
your tests in an environment variable. The temporary dir will be deleted
|
|
||||||
afterwards and the return value will be preserved.
|
|
||||||
"""
|
|
||||||
|
|
||||||
lock_dir = tempfile.mkdtemp()
|
|
||||||
os.environ["MURANO_LOCK_PATH"] = lock_dir
|
|
||||||
try:
|
|
||||||
ret_val = subprocess.call(argv[1:])
|
|
||||||
finally:
|
|
||||||
shutil.rmtree(lock_dir, ignore_errors=True)
|
|
||||||
return ret_val
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
sys.exit(main(sys.argv))
|
|
|
@ -15,6 +15,7 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
import logging
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
@ -22,7 +23,6 @@ from eventlet import event
|
||||||
from eventlet import greenthread
|
from eventlet import greenthread
|
||||||
|
|
||||||
from murano.openstack.common._i18n import _LE, _LW
|
from murano.openstack.common._i18n import _LE, _LW
|
||||||
from murano.openstack.common import log as logging
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
@ -84,9 +84,9 @@ class FixedIntervalLoopingCall(LoopingCallBase):
|
||||||
break
|
break
|
||||||
delay = end - start - interval
|
delay = end - start - interval
|
||||||
if delay > 0:
|
if delay > 0:
|
||||||
LOG.warn(_LW('task %(func_name)s run outlasted '
|
LOG.warning(_LW('task %(func_name)r run outlasted '
|
||||||
'interval by %(delay).2f sec'),
|
'interval by %(delay).2f sec'),
|
||||||
{'func_name': repr(self.f), 'delay': delay})
|
{'func_name': self.f, 'delay': delay})
|
||||||
greenthread.sleep(-delay if delay < 0 else 0)
|
greenthread.sleep(-delay if delay < 0 else 0)
|
||||||
except LoopingCallDone as e:
|
except LoopingCallDone as e:
|
||||||
self.stop()
|
self.stop()
|
||||||
|
@ -127,9 +127,9 @@ class DynamicLoopingCall(LoopingCallBase):
|
||||||
|
|
||||||
if periodic_interval_max is not None:
|
if periodic_interval_max is not None:
|
||||||
idle = min(idle, periodic_interval_max)
|
idle = min(idle, periodic_interval_max)
|
||||||
LOG.debug('Dynamic looping call %(func_name)s sleeping '
|
LOG.debug('Dynamic looping call %(func_name)r sleeping '
|
||||||
'for %(idle).02f seconds',
|
'for %(idle).02f seconds',
|
||||||
{'func_name': repr(self.f), 'idle': idle})
|
{'func_name': self.f, 'idle': idle})
|
||||||
greenthread.sleep(idle)
|
greenthread.sleep(idle)
|
||||||
except LoopingCallDone as e:
|
except LoopingCallDone as e:
|
||||||
self.stop()
|
self.stop()
|
||||||
|
|
|
@ -1,289 +0,0 @@
|
||||||
# Copyright 2011 OpenStack Foundation.
|
|
||||||
# All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
"""
|
|
||||||
System-level utilities and helper functions.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import errno
|
|
||||||
import logging
|
|
||||||
import multiprocessing
|
|
||||||
import os
|
|
||||||
import random
|
|
||||||
import shlex
|
|
||||||
import signal
|
|
||||||
|
|
||||||
from eventlet.green import subprocess
|
|
||||||
from eventlet import greenthread
|
|
||||||
from oslo_utils import strutils
|
|
||||||
import six
|
|
||||||
|
|
||||||
from murano.openstack.common._i18n import _
|
|
||||||
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class InvalidArgumentError(Exception):
|
|
||||||
def __init__(self, message=None):
|
|
||||||
super(InvalidArgumentError, self).__init__(message)
|
|
||||||
|
|
||||||
|
|
||||||
class UnknownArgumentError(Exception):
|
|
||||||
def __init__(self, message=None):
|
|
||||||
super(UnknownArgumentError, self).__init__(message)
|
|
||||||
|
|
||||||
|
|
||||||
class ProcessExecutionError(Exception):
|
|
||||||
def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
|
|
||||||
description=None):
|
|
||||||
self.exit_code = exit_code
|
|
||||||
self.stderr = stderr
|
|
||||||
self.stdout = stdout
|
|
||||||
self.cmd = cmd
|
|
||||||
self.description = description
|
|
||||||
|
|
||||||
if description is None:
|
|
||||||
description = _("Unexpected error while running command.")
|
|
||||||
if exit_code is None:
|
|
||||||
exit_code = '-'
|
|
||||||
message = _('%(description)s\n'
|
|
||||||
'Command: %(cmd)s\n'
|
|
||||||
'Exit code: %(exit_code)s\n'
|
|
||||||
'Stdout: %(stdout)r\n'
|
|
||||||
'Stderr: %(stderr)r') % {'description': description,
|
|
||||||
'cmd': cmd,
|
|
||||||
'exit_code': exit_code,
|
|
||||||
'stdout': stdout,
|
|
||||||
'stderr': stderr}
|
|
||||||
super(ProcessExecutionError, self).__init__(message)
|
|
||||||
|
|
||||||
|
|
||||||
class NoRootWrapSpecified(Exception):
|
|
||||||
def __init__(self, message=None):
|
|
||||||
super(NoRootWrapSpecified, self).__init__(message)
|
|
||||||
|
|
||||||
|
|
||||||
def _subprocess_setup():
|
|
||||||
# Python installs a SIGPIPE handler by default. This is usually not what
|
|
||||||
# non-Python subprocesses expect.
|
|
||||||
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
|
|
||||||
|
|
||||||
|
|
||||||
def execute(*cmd, **kwargs):
|
|
||||||
"""Helper method to shell out and execute a command through subprocess.
|
|
||||||
|
|
||||||
Allows optional retry.
|
|
||||||
|
|
||||||
:param cmd: Passed to subprocess.Popen.
|
|
||||||
:type cmd: string
|
|
||||||
:param process_input: Send to opened process.
|
|
||||||
:type process_input: string
|
|
||||||
:param env_variables: Environment variables and their values that
|
|
||||||
will be set for the process.
|
|
||||||
:type env_variables: dict
|
|
||||||
:param check_exit_code: Single bool, int, or list of allowed exit
|
|
||||||
codes. Defaults to [0]. Raise
|
|
||||||
:class:`ProcessExecutionError` unless
|
|
||||||
program exits with one of these code.
|
|
||||||
:type check_exit_code: boolean, int, or [int]
|
|
||||||
:param delay_on_retry: True | False. Defaults to True. If set to True,
|
|
||||||
wait a short amount of time before retrying.
|
|
||||||
:type delay_on_retry: boolean
|
|
||||||
:param attempts: How many times to retry cmd.
|
|
||||||
:type attempts: int
|
|
||||||
:param run_as_root: True | False. Defaults to False. If set to True,
|
|
||||||
the command is prefixed by the command specified
|
|
||||||
in the root_helper kwarg.
|
|
||||||
:type run_as_root: boolean
|
|
||||||
:param root_helper: command to prefix to commands called with
|
|
||||||
run_as_root=True
|
|
||||||
:type root_helper: string
|
|
||||||
:param shell: whether or not there should be a shell used to
|
|
||||||
execute this command. Defaults to false.
|
|
||||||
:type shell: boolean
|
|
||||||
:param loglevel: log level for execute commands.
|
|
||||||
:type loglevel: int. (Should be logging.DEBUG or logging.INFO)
|
|
||||||
:returns: (stdout, stderr) from process execution
|
|
||||||
:raises: :class:`UnknownArgumentError` on
|
|
||||||
receiving unknown arguments
|
|
||||||
:raises: :class:`ProcessExecutionError`
|
|
||||||
"""
|
|
||||||
|
|
||||||
process_input = kwargs.pop('process_input', None)
|
|
||||||
env_variables = kwargs.pop('env_variables', None)
|
|
||||||
check_exit_code = kwargs.pop('check_exit_code', [0])
|
|
||||||
ignore_exit_code = False
|
|
||||||
delay_on_retry = kwargs.pop('delay_on_retry', True)
|
|
||||||
attempts = kwargs.pop('attempts', 1)
|
|
||||||
run_as_root = kwargs.pop('run_as_root', False)
|
|
||||||
root_helper = kwargs.pop('root_helper', '')
|
|
||||||
shell = kwargs.pop('shell', False)
|
|
||||||
loglevel = kwargs.pop('loglevel', logging.DEBUG)
|
|
||||||
|
|
||||||
if isinstance(check_exit_code, bool):
|
|
||||||
ignore_exit_code = not check_exit_code
|
|
||||||
check_exit_code = [0]
|
|
||||||
elif isinstance(check_exit_code, int):
|
|
||||||
check_exit_code = [check_exit_code]
|
|
||||||
|
|
||||||
if kwargs:
|
|
||||||
raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs)
|
|
||||||
|
|
||||||
if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
|
|
||||||
if not root_helper:
|
|
||||||
raise NoRootWrapSpecified(
|
|
||||||
message=_('Command requested root, but did not '
|
|
||||||
'specify a root helper.'))
|
|
||||||
cmd = shlex.split(root_helper) + list(cmd)
|
|
||||||
|
|
||||||
cmd = map(str, cmd)
|
|
||||||
sanitized_cmd = strutils.mask_password(' '.join(cmd))
|
|
||||||
|
|
||||||
while attempts > 0:
|
|
||||||
attempts -= 1
|
|
||||||
try:
|
|
||||||
LOG.log(loglevel, _('Running cmd (subprocess): %s'), sanitized_cmd)
|
|
||||||
_PIPE = subprocess.PIPE # pylint: disable=E1101
|
|
||||||
|
|
||||||
if os.name == 'nt':
|
|
||||||
preexec_fn = None
|
|
||||||
close_fds = False
|
|
||||||
else:
|
|
||||||
preexec_fn = _subprocess_setup
|
|
||||||
close_fds = True
|
|
||||||
|
|
||||||
obj = subprocess.Popen(cmd,
|
|
||||||
stdin=_PIPE,
|
|
||||||
stdout=_PIPE,
|
|
||||||
stderr=_PIPE,
|
|
||||||
close_fds=close_fds,
|
|
||||||
preexec_fn=preexec_fn,
|
|
||||||
shell=shell,
|
|
||||||
env=env_variables)
|
|
||||||
result = None
|
|
||||||
for _i in six.moves.range(20):
|
|
||||||
# NOTE(russellb) 20 is an arbitrary number of retries to
|
|
||||||
# prevent any chance of looping forever here.
|
|
||||||
try:
|
|
||||||
if process_input is not None:
|
|
||||||
result = obj.communicate(process_input)
|
|
||||||
else:
|
|
||||||
result = obj.communicate()
|
|
||||||
except OSError as e:
|
|
||||||
if e.errno in (errno.EAGAIN, errno.EINTR):
|
|
||||||
continue
|
|
||||||
raise
|
|
||||||
break
|
|
||||||
obj.stdin.close() # pylint: disable=E1101
|
|
||||||
_returncode = obj.returncode # pylint: disable=E1101
|
|
||||||
LOG.log(loglevel, _('Result was %s'), _returncode)
|
|
||||||
if not ignore_exit_code and _returncode not in check_exit_code:
|
|
||||||
(stdout, stderr) = result
|
|
||||||
sanitized_stdout = strutils.mask_password(stdout)
|
|
||||||
sanitized_stderr = strutils.mask_password(stderr)
|
|
||||||
raise ProcessExecutionError(exit_code=_returncode,
|
|
||||||
stdout=sanitized_stdout,
|
|
||||||
stderr=sanitized_stderr,
|
|
||||||
cmd=sanitized_cmd)
|
|
||||||
return result
|
|
||||||
except ProcessExecutionError:
|
|
||||||
if not attempts:
|
|
||||||
raise
|
|
||||||
else:
|
|
||||||
LOG.log(loglevel, _('%r failed. Retrying.'), sanitized_cmd)
|
|
||||||
if delay_on_retry:
|
|
||||||
greenthread.sleep(random.randint(20, 200) / 100.0)
|
|
||||||
finally:
|
|
||||||
# NOTE(termie): this appears to be necessary to let the subprocess
|
|
||||||
# call clean something up in between calls, without
|
|
||||||
# it two execute calls in a row hangs the second one
|
|
||||||
greenthread.sleep(0)
|
|
||||||
|
|
||||||
|
|
||||||
def trycmd(*args, **kwargs):
|
|
||||||
"""A wrapper around execute() to more easily handle warnings and errors.
|
|
||||||
|
|
||||||
Returns an (out, err) tuple of strings containing the output of
|
|
||||||
the command's stdout and stderr. If 'err' is not empty then the
|
|
||||||
command can be considered to have failed.
|
|
||||||
|
|
||||||
:discard_warnings True | False. Defaults to False. If set to True,
|
|
||||||
then for succeeding commands, stderr is cleared
|
|
||||||
|
|
||||||
"""
|
|
||||||
discard_warnings = kwargs.pop('discard_warnings', False)
|
|
||||||
|
|
||||||
try:
|
|
||||||
out, err = execute(*args, **kwargs)
|
|
||||||
failed = False
|
|
||||||
except ProcessExecutionError as exn:
|
|
||||||
out, err = '', six.text_type(exn)
|
|
||||||
failed = True
|
|
||||||
|
|
||||||
if not failed and discard_warnings and err:
|
|
||||||
# Handle commands that output to stderr but otherwise succeed
|
|
||||||
err = ''
|
|
||||||
|
|
||||||
return out, err
|
|
||||||
|
|
||||||
|
|
||||||
def ssh_execute(ssh, cmd, process_input=None,
|
|
||||||
addl_env=None, check_exit_code=True):
|
|
||||||
sanitized_cmd = strutils.mask_password(cmd)
|
|
||||||
LOG.debug('Running cmd (SSH): %s', sanitized_cmd)
|
|
||||||
if addl_env:
|
|
||||||
raise InvalidArgumentError(_('Environment not supported over SSH'))
|
|
||||||
|
|
||||||
if process_input:
|
|
||||||
# This is (probably) fixable if we need it...
|
|
||||||
raise InvalidArgumentError(_('process_input not supported over SSH'))
|
|
||||||
|
|
||||||
stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd)
|
|
||||||
channel = stdout_stream.channel
|
|
||||||
|
|
||||||
# NOTE(justinsb): This seems suspicious...
|
|
||||||
# ...other SSH clients have buffering issues with this approach
|
|
||||||
stdout = stdout_stream.read()
|
|
||||||
sanitized_stdout = strutils.mask_password(stdout)
|
|
||||||
stderr = stderr_stream.read()
|
|
||||||
sanitized_stderr = strutils.mask_password(stderr)
|
|
||||||
|
|
||||||
stdin_stream.close()
|
|
||||||
|
|
||||||
exit_status = channel.recv_exit_status()
|
|
||||||
|
|
||||||
# exit_status == -1 if no exit code was returned
|
|
||||||
if exit_status != -1:
|
|
||||||
LOG.debug('Result was %s' % exit_status)
|
|
||||||
if check_exit_code and exit_status != 0:
|
|
||||||
raise ProcessExecutionError(exit_code=exit_status,
|
|
||||||
stdout=sanitized_stdout,
|
|
||||||
stderr=sanitized_stderr,
|
|
||||||
cmd=sanitized_cmd)
|
|
||||||
|
|
||||||
return (sanitized_stdout, sanitized_stderr)
|
|
||||||
|
|
||||||
|
|
||||||
def get_worker_count():
|
|
||||||
"""Utility to get the default worker count.
|
|
||||||
|
|
||||||
@return: The number of CPUs if that can be determined, else a default
|
|
||||||
worker count of 1 is returned.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
return multiprocessing.cpu_count()
|
|
||||||
except NotImplementedError:
|
|
||||||
return 1
|
|
|
@ -18,28 +18,20 @@
|
||||||
"""Generic Node base class for all workers that run on hosts."""
|
"""Generic Node base class for all workers that run on hosts."""
|
||||||
|
|
||||||
import errno
|
import errno
|
||||||
import logging as std_logging
|
import io
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
import signal
|
import signal
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
|
|
||||||
try:
|
|
||||||
# Importing just the symbol here because the io module does not
|
|
||||||
# exist in Python 2.6.
|
|
||||||
from io import UnsupportedOperation # noqa
|
|
||||||
except ImportError:
|
|
||||||
# Python 2.6
|
|
||||||
UnsupportedOperation = None
|
|
||||||
|
|
||||||
import eventlet
|
import eventlet
|
||||||
from eventlet import event
|
from eventlet import event
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
|
|
||||||
from murano.openstack.common import eventlet_backdoor
|
from murano.openstack.common import eventlet_backdoor
|
||||||
from murano.openstack.common._i18n import _LE, _LI, _LW
|
from murano.openstack.common._i18n import _LE, _LI, _LW
|
||||||
from murano.openstack.common import log as logging
|
|
||||||
from murano.openstack.common import systemd
|
from murano.openstack.common import systemd
|
||||||
from murano.openstack.common import threadgroup
|
from murano.openstack.common import threadgroup
|
||||||
|
|
||||||
|
@ -60,15 +52,15 @@ def _is_daemon():
|
||||||
# http://www.gnu.org/software/bash/manual/bashref.html#Job-Control-Basics
|
# http://www.gnu.org/software/bash/manual/bashref.html#Job-Control-Basics
|
||||||
try:
|
try:
|
||||||
is_daemon = os.getpgrp() != os.tcgetpgrp(sys.stdout.fileno())
|
is_daemon = os.getpgrp() != os.tcgetpgrp(sys.stdout.fileno())
|
||||||
|
except io.UnsupportedOperation:
|
||||||
|
# Could not get the fileno for stdout, so we must be a daemon.
|
||||||
|
is_daemon = True
|
||||||
except OSError as err:
|
except OSError as err:
|
||||||
if err.errno == errno.ENOTTY:
|
if err.errno == errno.ENOTTY:
|
||||||
# Assume we are a daemon because there is no terminal.
|
# Assume we are a daemon because there is no terminal.
|
||||||
is_daemon = True
|
is_daemon = True
|
||||||
else:
|
else:
|
||||||
raise
|
raise
|
||||||
except UnsupportedOperation:
|
|
||||||
# Could not get the fileno for stdout, so we must be a daemon.
|
|
||||||
is_daemon = True
|
|
||||||
return is_daemon
|
return is_daemon
|
||||||
|
|
||||||
|
|
||||||
|
@ -163,7 +155,7 @@ class ServiceLauncher(Launcher):
|
||||||
signo = 0
|
signo = 0
|
||||||
|
|
||||||
LOG.debug('Full set of CONF:')
|
LOG.debug('Full set of CONF:')
|
||||||
CONF.log_opt_values(LOG, std_logging.DEBUG)
|
CONF.log_opt_values(LOG, logging.DEBUG)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if ready_callback:
|
if ready_callback:
|
||||||
|
@ -200,6 +192,13 @@ class ServiceWrapper(object):
|
||||||
|
|
||||||
|
|
||||||
class ProcessLauncher(object):
|
class ProcessLauncher(object):
|
||||||
|
_signal_handlers_set = set()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _handle_class_signals(cls, *args, **kwargs):
|
||||||
|
for handler in cls._signal_handlers_set:
|
||||||
|
handler(*args, **kwargs)
|
||||||
|
|
||||||
def __init__(self, wait_interval=0.01):
|
def __init__(self, wait_interval=0.01):
|
||||||
"""Constructor.
|
"""Constructor.
|
||||||
|
|
||||||
|
@ -215,7 +214,8 @@ class ProcessLauncher(object):
|
||||||
self.handle_signal()
|
self.handle_signal()
|
||||||
|
|
||||||
def handle_signal(self):
|
def handle_signal(self):
|
||||||
_set_signals_handler(self._handle_signal)
|
self._signal_handlers_set.add(self._handle_signal)
|
||||||
|
_set_signals_handler(self._handle_class_signals)
|
||||||
|
|
||||||
def _handle_signal(self, signo, frame):
|
def _handle_signal(self, signo, frame):
|
||||||
self.sigcaught = signo
|
self.sigcaught = signo
|
||||||
|
@ -227,7 +227,7 @@ class ProcessLauncher(object):
|
||||||
def _pipe_watcher(self):
|
def _pipe_watcher(self):
|
||||||
# This will block until the write end is closed when the parent
|
# This will block until the write end is closed when the parent
|
||||||
# dies unexpectedly
|
# dies unexpectedly
|
||||||
self.readpipe.read()
|
self.readpipe.read(1)
|
||||||
|
|
||||||
LOG.info(_LI('Parent process has died unexpectedly, exiting'))
|
LOG.info(_LI('Parent process has died unexpectedly, exiting'))
|
||||||
|
|
||||||
|
@ -235,15 +235,12 @@ class ProcessLauncher(object):
|
||||||
|
|
||||||
def _child_process_handle_signal(self):
|
def _child_process_handle_signal(self):
|
||||||
# Setup child signal handlers differently
|
# Setup child signal handlers differently
|
||||||
def _sigterm(*args):
|
|
||||||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
|
||||||
raise SignalExit(signal.SIGTERM)
|
|
||||||
|
|
||||||
def _sighup(*args):
|
def _sighup(*args):
|
||||||
signal.signal(signal.SIGHUP, signal.SIG_DFL)
|
signal.signal(signal.SIGHUP, signal.SIG_DFL)
|
||||||
raise SignalExit(signal.SIGHUP)
|
raise SignalExit(signal.SIGHUP)
|
||||||
|
|
||||||
signal.signal(signal.SIGTERM, _sigterm)
|
# Parent signals with SIGTERM when it wants us to go away.
|
||||||
|
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||||
if _sighup_supported():
|
if _sighup_supported():
|
||||||
signal.signal(signal.SIGHUP, _sighup)
|
signal.signal(signal.SIGHUP, _sighup)
|
||||||
# Block SIGINT and let the parent send us a SIGTERM
|
# Block SIGINT and let the parent send us a SIGTERM
|
||||||
|
@ -377,7 +374,7 @@ class ProcessLauncher(object):
|
||||||
|
|
||||||
systemd.notify_once()
|
systemd.notify_once()
|
||||||
LOG.debug('Full set of CONF:')
|
LOG.debug('Full set of CONF:')
|
||||||
CONF.log_opt_values(LOG, std_logging.DEBUG)
|
CONF.log_opt_values(LOG, logging.DEBUG)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
while True:
|
while True:
|
||||||
|
@ -392,12 +389,18 @@ class ProcessLauncher(object):
|
||||||
if not _is_sighup_and_daemon(self.sigcaught):
|
if not _is_sighup_and_daemon(self.sigcaught):
|
||||||
break
|
break
|
||||||
|
|
||||||
|
cfg.CONF.reload_config_files()
|
||||||
|
for service in set(
|
||||||
|
[wrap.service for wrap in self.children.values()]):
|
||||||
|
service.reset()
|
||||||
|
|
||||||
for pid in self.children:
|
for pid in self.children:
|
||||||
os.kill(pid, signal.SIGHUP)
|
os.kill(pid, signal.SIGHUP)
|
||||||
|
|
||||||
self.running = True
|
self.running = True
|
||||||
self.sigcaught = None
|
self.sigcaught = None
|
||||||
except eventlet.greenlet.GreenletExit:
|
except eventlet.greenlet.GreenletExit:
|
||||||
LOG.info(_LI("Wait called after thread killed. Cleaning up."))
|
LOG.info(_LI("Wait called after thread killed. Cleaning up."))
|
||||||
|
|
||||||
self.stop()
|
self.stop()
|
||||||
|
|
||||||
|
@ -434,8 +437,8 @@ class Service(object):
|
||||||
def start(self):
|
def start(self):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def stop(self):
|
def stop(self, graceful=False):
|
||||||
self.tg.stop()
|
self.tg.stop(graceful)
|
||||||
self.tg.wait()
|
self.tg.wait()
|
||||||
# Signal that service cleanup is done:
|
# Signal that service cleanup is done:
|
||||||
if not self._done.ready():
|
if not self._done.ready():
|
||||||
|
|
|
@ -39,7 +39,7 @@ CONF.register_opts(ssl_opts, config_section)
|
||||||
|
|
||||||
|
|
||||||
def list_opts():
|
def list_opts():
|
||||||
"""Entry point for oslo.config-generator."""
|
"""Entry point for oslo-config-generator."""
|
||||||
return [(config_section, copy.deepcopy(ssl_opts))]
|
return [(config_section, copy.deepcopy(ssl_opts))]
|
||||||
|
|
||||||
|
|
||||||
|
@ -79,23 +79,3 @@ def wrap(sock):
|
||||||
ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED
|
ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED
|
||||||
|
|
||||||
return ssl.wrap_socket(sock, **ssl_kwargs)
|
return ssl.wrap_socket(sock, **ssl_kwargs)
|
||||||
|
|
||||||
|
|
||||||
_SSL_PROTOCOLS = {
|
|
||||||
"tlsv1": ssl.PROTOCOL_TLSv1,
|
|
||||||
"sslv23": ssl.PROTOCOL_SSLv23,
|
|
||||||
"sslv3": ssl.PROTOCOL_SSLv3
|
|
||||||
}
|
|
||||||
|
|
||||||
try:
|
|
||||||
_SSL_PROTOCOLS["sslv2"] = ssl.PROTOCOL_SSLv2
|
|
||||||
except AttributeError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def validate_ssl_version(version):
|
|
||||||
key = version.lower()
|
|
||||||
try:
|
|
||||||
return _SSL_PROTOCOLS[key]
|
|
||||||
except KeyError:
|
|
||||||
raise RuntimeError(_("Invalid SSL version : %s") % version)
|
|
||||||
|
|
|
@ -16,12 +16,11 @@
|
||||||
Helper module for systemd service readiness notification.
|
Helper module for systemd service readiness notification.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import socket
|
import socket
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from murano.openstack.common import log as logging
|
|
||||||
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
@ -11,12 +11,13 @@
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
import logging
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
import eventlet
|
import eventlet
|
||||||
from eventlet import greenpool
|
from eventlet import greenpool
|
||||||
|
|
||||||
from murano.openstack.common import log as logging
|
from murano.openstack.common._i18n import _LE
|
||||||
from murano.openstack.common import loopingcall
|
from murano.openstack.common import loopingcall
|
||||||
|
|
||||||
|
|
||||||
|
@ -98,15 +99,15 @@ class ThreadGroup(object):
|
||||||
x.stop()
|
x.stop()
|
||||||
except eventlet.greenlet.GreenletExit:
|
except eventlet.greenlet.GreenletExit:
|
||||||
pass
|
pass
|
||||||
except Exception as ex:
|
except Exception:
|
||||||
LOG.exception(ex)
|
LOG.exception(_LE('Error stopping thread.'))
|
||||||
|
|
||||||
def stop_timers(self):
|
def stop_timers(self):
|
||||||
for x in self.timers:
|
for x in self.timers:
|
||||||
try:
|
try:
|
||||||
x.stop()
|
x.stop()
|
||||||
except Exception as ex:
|
except Exception:
|
||||||
LOG.exception(ex)
|
LOG.exception(_LE('Error stopping timer.'))
|
||||||
self.timers = []
|
self.timers = []
|
||||||
|
|
||||||
def stop(self, graceful=False):
|
def stop(self, graceful=False):
|
||||||
|
@ -132,8 +133,8 @@ class ThreadGroup(object):
|
||||||
x.wait()
|
x.wait()
|
||||||
except eventlet.greenlet.GreenletExit:
|
except eventlet.greenlet.GreenletExit:
|
||||||
pass
|
pass
|
||||||
except Exception as ex:
|
except Exception:
|
||||||
LOG.exception(ex)
|
LOG.exception(_LE('Error waiting on ThreadGroup.'))
|
||||||
current = threading.current_thread()
|
current = threading.current_thread()
|
||||||
|
|
||||||
# Iterate over a copy of self.threads so thread_done doesn't
|
# Iterate over a copy of self.threads so thread_done doesn't
|
||||||
|
|
|
@ -13,7 +13,7 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
import murano.openstack.common.exception as e
|
import murano.common.exceptions as e
|
||||||
|
|
||||||
|
|
||||||
class PackageException(e.Error):
|
class PackageException(e.Error):
|
||||||
|
|
|
@ -1,8 +1,6 @@
|
||||||
[DEFAULT]
|
[DEFAULT]
|
||||||
|
|
||||||
# The list of modules to copy from openstack-common
|
# The list of modules to copy from openstack-common
|
||||||
module=exception
|
|
||||||
module=lockutils
|
|
||||||
module=log
|
module=log
|
||||||
module=service
|
module=service
|
||||||
module=sslutils
|
module=sslutils
|
||||||
|
|
Loading…
Reference in New Issue