282 lines
9.1 KiB
Python
282 lines
9.1 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# It's based on oslo.i18n usage in OpenStack Keystone project and
|
|
# recommendations from
|
|
# https://docs.openstack.org/oslo.i18n/latest/user/usage.html
|
|
|
|
"""Utilities and helper functions."""
|
|
import base64
|
|
import binascii
|
|
import eventlet
|
|
import functools
|
|
import inspect
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
import zipfile
|
|
import ctypes, os
|
|
|
|
from oslo_concurrency import lockutils
|
|
from oslo_concurrency import processutils
|
|
from oslo_utils import importutils
|
|
from oslo_context import context as common_context
|
|
from oslo_log import log as logging
|
|
from oslo_utils import excutils
|
|
from oslo_utils import strutils
|
|
import pecan
|
|
import six
|
|
|
|
from gyan.api import utils as api_utils
|
|
from gyan.common import consts
|
|
from gyan.common import exception
|
|
from gyan.common.i18n import _
|
|
from gyan.common import privileged
|
|
import gyan.conf
|
|
from gyan import objects
|
|
|
|
eventlet.monkey_patch()
|
|
|
|
CONF = gyan.conf.CONF
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
synchronized = lockutils.synchronized_with_prefix(consts.NAME_PREFIX)
|
|
|
|
VALID_STATES = {
|
|
'deploy': [consts.CREATED, consts.UNDEPLOYED, consts.SCHEDULED],
|
|
'undeploy': [consts.DEPLOYED, consts.DEPLOYED_COMPUTE_NODE, consts.DEPLOYMENT_FAILED, consts.DEPLOYMENT_STARTED]
|
|
}
|
|
def safe_rstrip(value, chars=None):
|
|
"""Removes trailing characters from a string if that does not make it empty
|
|
|
|
:param value: A string value that will be stripped.
|
|
:param chars: Characters to remove.
|
|
:return: Stripped value.
|
|
|
|
"""
|
|
if not isinstance(value, six.string_types):
|
|
LOG.warning(
|
|
"Failed to remove trailing character. Returning original object. "
|
|
"Supplied object is not a string: %s.", value)
|
|
return value
|
|
|
|
return value.rstrip(chars) or value
|
|
|
|
|
|
def _do_allow_certain_content_types(func, content_types_list):
|
|
# Allows you to bypass pecan's content-type restrictions
|
|
cfg = pecan.util._cfg(func)
|
|
cfg.setdefault('content_types', {})
|
|
cfg['content_types'].update((value, '')
|
|
for value in content_types_list)
|
|
return func
|
|
|
|
|
|
def allow_certain_content_types(*content_types_list):
|
|
def _wrapper(func):
|
|
return _do_allow_certain_content_types(func, content_types_list)
|
|
return _wrapper
|
|
|
|
|
|
def allow_all_content_types(f):
|
|
return _do_allow_certain_content_types(f, mimetypes.types_map.values())
|
|
|
|
|
|
def spawn_n(func, *args, **kwargs):
|
|
"""Passthrough method for eventlet.spawn_n.
|
|
|
|
This utility exists so that it can be stubbed for testing without
|
|
interfering with the service spawns.
|
|
|
|
It will also grab the context from the threadlocal store and add it to
|
|
the store on the new thread. This allows for continuity in logging the
|
|
context when using this method to spawn a new thread.
|
|
"""
|
|
_context = common_context.get_current()
|
|
|
|
@functools.wraps(func)
|
|
def context_wrapper(*args, **kwargs):
|
|
# NOTE: If update_store is not called after spawn_n it won't be
|
|
# available for the logger to pull from threadlocal storage.
|
|
_context.update_store()
|
|
if _context is not None:
|
|
_context.update_store()
|
|
func(*args, **kwargs)
|
|
|
|
eventlet.spawn_n(context_wrapper, *args, **kwargs)
|
|
|
|
|
|
def translate_exception(function):
|
|
"""Wraps a method to catch exceptions.
|
|
|
|
If the exception is not an instance of GyanException,
|
|
translate it into one.
|
|
"""
|
|
|
|
@functools.wraps(function)
|
|
def decorated_function(self, context, *args, **kwargs):
|
|
try:
|
|
return function(self, context, *args, **kwargs)
|
|
except Exception as e:
|
|
if not isinstance(e, exception.GyanException):
|
|
LOG.exception("Unexpected error: %s", six.text_type(e))
|
|
e = exception.GyanException("Unexpected error: %s"
|
|
% six.text_type(e))
|
|
raise e
|
|
raise
|
|
|
|
return decorated_function
|
|
|
|
|
|
def custom_execute(*cmd, **kwargs):
|
|
try:
|
|
return processutils.execute(*cmd, **kwargs)
|
|
except processutils.ProcessExecutionError as e:
|
|
sanitized_cmd = strutils.mask_password(' '.join(cmd))
|
|
raise exception.CommandError(cmd=sanitized_cmd,
|
|
error=six.text_type(e))
|
|
|
|
|
|
def is_all_projects(search_opts):
|
|
all_projects = search_opts.get('all_projects')
|
|
if all_projects:
|
|
try:
|
|
all_projects = strutils.bool_from_string(all_projects, True)
|
|
except ValueError:
|
|
bools = ', '.join(strutils.TRUE_STRINGS + strutils.FALSE_STRINGS)
|
|
raise exception.InvalidValue(_('Valid all_projects values are: %s')
|
|
% bools)
|
|
else:
|
|
all_projects = False
|
|
return all_projects
|
|
|
|
|
|
def get_ml_model(ml_model_ident):
|
|
ml_model = api_utils.get_resource('ML_Model', ml_model_ident)
|
|
if not ml_model:
|
|
pecan.abort(404, ('Not found; the ml model you requested '
|
|
'does not exist.'))
|
|
|
|
return ml_model
|
|
|
|
def get_flavor(flavor_ident):
|
|
flavor = api_utils.get_resource('Flavor', flavor_ident)
|
|
if not flavor:
|
|
pecan.abort(404, ('Not found; the ml model you requested '
|
|
'does not exist.'))
|
|
|
|
return flavor
|
|
|
|
def validate_ml_model_state(ml_model, action):
|
|
if ml_model.status not in VALID_STATES[action]:
|
|
raise exception.InvalidStateException(
|
|
id=ml_model.id,
|
|
action=action,
|
|
actual_state=ml_model.status)
|
|
|
|
|
|
def get_wrapped_function(function):
|
|
"""Get the method at the bottom of a stack of decorators."""
|
|
if not hasattr(function, '__closure__') or not function.__closure__:
|
|
return function
|
|
|
|
def _get_wrapped_function(function):
|
|
if not hasattr(function, '__closure__') or not function.__closure__:
|
|
return None
|
|
|
|
for closure in function.__closure__:
|
|
func = closure.cell_contents
|
|
|
|
deeper_func = _get_wrapped_function(func)
|
|
if deeper_func:
|
|
return deeper_func
|
|
elif hasattr(closure.cell_contents, '__call__'):
|
|
return closure.cell_contents
|
|
|
|
return function
|
|
|
|
return _get_wrapped_function(function)
|
|
|
|
|
|
def wrap_ml_model_event(prefix):
|
|
"""Warps a method to log the event taken on the ml_model, and result.
|
|
|
|
This decorator wraps a method to log the start and result of an event, as
|
|
part of an action taken on a ml_model.
|
|
"""
|
|
def helper(function):
|
|
|
|
@functools.wraps(function)
|
|
def decorated_function(self, context, *args, **kwargs):
|
|
wrapped_func = get_wrapped_function(function)
|
|
keyed_args = inspect.getcallargs(wrapped_func, self, context,
|
|
*args, **kwargs)
|
|
ml_model_uuid = keyed_args['ml_model'].uuid
|
|
|
|
event_name = '{0}_{1}'.format(prefix, function.__name__)
|
|
with EventReporter(context, event_name, ml_model_uuid):
|
|
return function(self, context, *args, **kwargs)
|
|
return decorated_function
|
|
return helper
|
|
|
|
|
|
def wrap_exception():
|
|
def helper(function):
|
|
|
|
@functools.wraps(function)
|
|
def decorated_function(self, context, ml_model, *args, **kwargs):
|
|
try:
|
|
return function(self, context, ml_model, *args, **kwargs)
|
|
except exception.DockerError as e:
|
|
with excutils.save_and_reraise_exception(reraise=False):
|
|
LOG.error("Error occurred while calling Docker API: %s",
|
|
six.text_type(e))
|
|
except Exception as e:
|
|
with excutils.save_and_reraise_exception(reraise=False):
|
|
LOG.exception("Unexpected exception: %s", six.text_type(e))
|
|
return decorated_function
|
|
return helper
|
|
|
|
|
|
def is_close(x, y, rel_tol=1e-06, abs_tol=0.0):
|
|
return abs(x - y) <= max(rel_tol * max(abs(x), abs(y)), abs_tol)
|
|
|
|
|
|
def is_less_than(x, y):
|
|
if isinstance(x, int) and isinstance(y, int):
|
|
return x < y
|
|
if isinstance(x, float) or isinstance(y, float):
|
|
return False if (x - y) >= 0 or is_close(x, y) else True
|
|
|
|
|
|
def encode_file_data(data):
|
|
if six.PY3 and isinstance(data, str):
|
|
data = data.encode('utf-8')
|
|
return base64.b64encode(data).decode('utf-8')
|
|
|
|
|
|
def decode_file_data(data):
|
|
# Py3 raises binascii.Error instead of TypeError as in Py27
|
|
try:
|
|
return base64.b64decode(data)
|
|
except (TypeError, binascii.Error):
|
|
raise exception.Base64Exception()
|
|
|
|
|
|
def save_model(path, model):
|
|
file_path = os.path.join(path, model.id)
|
|
with open(file_path+'.zip', 'wb') as f:
|
|
f.write(model.ml_data)
|
|
zip_ref = zipfile.ZipFile(file_path+'.zip', 'r')
|
|
zip_ref.extractall(file_path)
|
|
zip_ref.close() |