moved generic util functions from mistral to mistral-lib

Depends-On: I780c270e4b1a184d7d4dcc580d23697ba75edab1
Closes-bug: #1815183
Change-Id: I5a1d402baa3f69c37f9347c8b3d02a83b8f60423
This commit is contained in:
ali 2019-08-14 09:03:30 +00:00 committed by Renat Akhmerov
parent 9585a63847
commit 7e7f1cb92b
62 changed files with 63 additions and 717 deletions

View File

@ -50,7 +50,7 @@ logutils==0.3.5
Mako==0.4.0
MarkupSafe==1.0
mccabe==0.2.1
mistral-lib==0.4.0
mistral-lib==1.2.0
mock==2.0.0
monotonic==0.6
mox3==0.20.0

View File

@ -16,7 +16,7 @@ import json
from wsme import types as wtypes
from mistral import utils
from mistral_lib import utils
class Resource(wtypes.Base):

View File

@ -25,9 +25,9 @@ from mistral.api.controllers.v2 import types
from mistral import context
from mistral.db.v2 import api as db_api
from mistral import exceptions
from mistral.utils import cut
from mistral.utils import filter_utils
from mistral.utils import rest_utils
from mistral_lib.utils import cut
LOG = logging.getLogger(__name__)

View File

@ -34,9 +34,9 @@ from mistral import exceptions as exc
from mistral.rpc import clients as rpc
from mistral.services import workflows as wf_service
from mistral.utils import filter_utils
from mistral.utils import merge_dicts
from mistral.utils import rest_utils
from mistral.workflow import states
from mistral_lib.utils import merge_dicts
LOG = logging.getLogger(__name__)

View File

@ -19,8 +19,8 @@ from wsme import types as wtypes
from mistral.api.controllers import resource
from mistral.api.controllers.v2 import types
from mistral import exceptions as exc
from mistral import utils
from mistral.workflow import states
from mistral_lib import utils
SCOPE_TYPES = wtypes.Enum(str, 'private', 'public')

View File

@ -27,7 +27,7 @@ from pecan import hooks
from mistral import auth
from mistral import exceptions as exc
from mistral import utils
from mistral_lib import utils
CONF = cfg.CONF
_CTX_THREAD_LOCAL_NAME = "MISTRAL_APP_CTX_THREAD_LOCAL"

View File

@ -22,7 +22,7 @@ import sqlalchemy as sa
from mistral.db.sqlalchemy import sqlite_lock
from mistral import exceptions as exc
from mistral import utils
from mistral_lib import utils
# Note(dzimine): sqlite only works for basic testing.

View File

@ -27,7 +27,7 @@ down_revision = '026'
from alembic import op
import datetime
from mistral import utils
from mistral_lib import utils
from oslo_config import cfg
from sqlalchemy import Column, DateTime, Boolean

View File

@ -19,7 +19,7 @@ from sqlalchemy.ext import declarative
from sqlalchemy.orm import attributes
from mistral.services import security
from mistral import utils
from mistral_lib import utils
def id_column():

View File

@ -38,8 +38,8 @@ from mistral.db.v2.sqlalchemy import filters as db_filters
from mistral.db.v2.sqlalchemy import models
from mistral import exceptions as exc
from mistral.services import security
from mistral import utils
from mistral.workflow import states
from mistral_lib import utils
CONF = cfg.CONF

View File

@ -29,7 +29,7 @@ from mistral.db.sqlalchemy import model_base as mb
from mistral.db.sqlalchemy import types as st
from mistral import exceptions as exc
from mistral.services import security
from mistral import utils
from mistral_lib import utils
# Definition objects.

View File

@ -31,11 +31,11 @@ from mistral.lang import parser as spec_parser
from mistral.rpc import clients as rpc
from mistral.services import action_manager as a_m
from mistral.services import security
from mistral import utils
from mistral.utils import wf_trace
from mistral.workflow import data_flow
from mistral.workflow import states
from mistral_lib import actions as ml_actions
from mistral_lib import utils
LOG = logging.getLogger(__name__)

View File

@ -29,8 +29,8 @@ from mistral.engine import base
from mistral.engine import post_tx_queue
from mistral.engine import workflow_handler as wf_handler
from mistral import exceptions
from mistral import utils as u
from mistral.workflow import states
from mistral_lib import utils as u
# Submodules of mistral.engine will throw NoSuchOptError if configuration

View File

@ -22,8 +22,8 @@ from mistral.scheduler import base as sched_base
from mistral.service import base as service_base
from mistral.services import action_execution_checker
from mistral.services import expiration_policy
from mistral import utils
from mistral.utils import profiler as profiler_utils
from mistral_lib import utils
LOG = logging.getLogger(__name__)

View File

@ -20,7 +20,7 @@ from oslo_log import log as logging
from mistral import context
from mistral.db import utils as db_utils
from mistral.db.v2 import api as db_api
from mistral import utils
from mistral_lib import utils
"""

View File

@ -32,12 +32,12 @@ from mistral import exceptions as exc
from mistral import expressions as expr
from mistral.notifiers import base as notif
from mistral.notifiers import notification_events as events
from mistral import utils
from mistral.utils import wf_trace
from mistral.workflow import base as wf_base
from mistral.workflow import commands
from mistral.workflow import data_flow
from mistral.workflow import states
from mistral_lib import utils
LOG = logging.getLogger(__name__)

View File

@ -17,7 +17,7 @@ import copy
from mistral.db.v2 import api as db_api
from mistral import exceptions as exc
from mistral import utils
from mistral_lib import utils
def _compare_parameters(expected_input, actual_input):

View File

@ -34,14 +34,13 @@ from mistral.notifiers import notification_events as events
from mistral.rpc import clients as rpc
from mistral.services import triggers
from mistral.services import workflows as wf_service
from mistral import utils
from mistral.utils import merge_dicts
from mistral.utils import wf_trace
from mistral.workflow import base as wf_base
from mistral.workflow import commands
from mistral.workflow import data_flow
from mistral.workflow import states
from mistral_lib import actions as ml_actions
from mistral_lib import utils
LOG = logging.getLogger(__name__)
@ -538,7 +537,7 @@ class Workflow(object):
cfg.CONF.engine.execution_field_size_limit_kb
)
self.wf_ex.output = merge_dicts({'result': msg}, output_on_error)
self.wf_ex.output = utils.merge_dicts({'result': msg}, output_on_error)
# Publish event.
self.notify(events.WORKFLOW_FAILED)

View File

@ -19,8 +19,8 @@ from mistral.executors import default_executor as exe
from mistral.rpc import base as rpc
from mistral.service import base as service_base
from mistral.services import action_execution_reporter
from mistral import utils
from mistral.utils import profiler as profiler_utils
from mistral_lib import utils
CONF = cfg.CONF
LOG = logging.getLogger(__name__)

View File

@ -26,8 +26,8 @@ from yaql.language import factory
from mistral.config import cfg
from mistral import exceptions as exc
from mistral.expressions.base_expression import Evaluator
from mistral import utils
from mistral.utils import expression_utils
from mistral_lib import utils
LOG = logging.getLogger(__name__)

View File

@ -24,7 +24,7 @@ from mistral import expressions as expr
from mistral.expressions.jinja_expression import ANY_JINJA_REGEXP
from mistral.expressions.yaql_expression import INLINE_YAQL_REGEXP
from mistral.lang import types
from mistral import utils
from mistral_lib import utils
ACTION_PATTERNS = {
"command": r"[\w\.]+[^=\(\s\"]*",

View File

@ -17,7 +17,7 @@ import six
from mistral.lang import types
from mistral.lang.v2 import base
from mistral import utils
from mistral_lib import utils
class ActionSpec(base.BaseSpec):

View File

@ -26,8 +26,8 @@ from mistral.lang.v2 import on_clause
from mistral.lang.v2 import policies
from mistral.lang.v2 import publish
from mistral.lang.v2 import retry_policy
from mistral import utils
from mistral.workflow import states
from mistral_lib import utils
_expr_ptrns = [expressions.patterns[name] for name in expressions.patterns]
WITH_ITEMS_PTRN = re.compile(

View File

@ -21,7 +21,7 @@ from mistral.lang import types
from mistral.lang.v2 import base
from mistral.lang.v2 import task_defaults
from mistral.lang.v2 import tasks
from mistral import utils
from mistral_lib import utils
NOOP_COMMAND = 'noop'
FAIL_COMMAND = 'fail'

View File

@ -18,8 +18,8 @@ from mistral import config as cfg
from mistral.notifiers import default_notifier as notif
from mistral.rpc import base as rpc
from mistral.service import base as service_base
from mistral import utils
from mistral.utils import profiler as profiler_utils
from mistral_lib import utils
LOG = logging.getLogger(__name__)

View File

@ -28,7 +28,7 @@ from mistral.rpc import base as rpc_base
from mistral.rpc.kombu import base as kombu_base
from mistral.rpc.kombu import kombu_hosts
from mistral.rpc.kombu import kombu_listener
from mistral import utils
from mistral_lib import utils
#: When connection to the RabbitMQ server breaks, the
#: client will receive EPIPE socket errors. These indicate

View File

@ -28,7 +28,7 @@ from mistral.db import utils as db_utils
from mistral.db.v2 import api as db_api
from mistral import exceptions as exc
from mistral.scheduler import base
from mistral import utils
from mistral_lib import utils
LOG = logging.getLogger(__name__)

View File

@ -21,7 +21,7 @@ from oslo_service import threadgroup
import tenacity
import tooz.coordination
from mistral import utils
from mistral_lib import utils
LOG = log.getLogger(__name__)

View File

@ -21,8 +21,8 @@ from mistral.db import utils as db_utils
from mistral.db.v2 import api as db_api
from mistral.engine import action_handler
from mistral.engine import post_tx_queue
from mistral import utils
from mistral_lib import actions as mistral_lib
from mistral_lib import utils
from oslo_config import cfg
from oslo_log import log as logging

View File

@ -21,8 +21,8 @@ from mistral.actions import generator_factory
from mistral.db.v2 import api as db_api
from mistral import exceptions as exc
from mistral.services import actions
from mistral import utils
from mistral.utils import inspect_utils as i_utils
from mistral_lib import utils
# TODO(rakhmerov): Make methods more consistent and granular.

View File

@ -29,7 +29,7 @@ from mistral.db import utils as db_utils
from mistral.db.v2 import api as db_api
from mistral import exceptions as exc
from mistral.scheduler import base as sched_base
from mistral import utils
from mistral_lib import utils
LOG = logging.getLogger(__name__)

View File

@ -16,8 +16,8 @@ import yaml
from mistral.db.v2 import api as db_api
from mistral import exceptions as exc
from mistral.lang import parser as spec_parser
from mistral import utils
from mistral.workflow import states
from mistral_lib import utils
from oslo_log import log as logging

View File

@ -18,7 +18,7 @@ import datetime
from mistral.api.controllers.v2 import resources
from mistral.db.v2 import api as db_api
from mistral.tests.unit import base
from mistral import utils
from mistral_lib import utils
WF_EXEC = {

View File

@ -21,7 +21,7 @@ from mistral.db.v2 import api as db_api
from mistral.db.v2.sqlalchemy import models
from mistral import exceptions as exc
from mistral.tests.unit.api import base
from mistral import utils
from mistral_lib import utils
ACTION_DEFINITION = """

View File

@ -36,9 +36,9 @@ from mistral.rpc import base as rpc_base
from mistral.rpc import clients as rpc_clients
from mistral.tests.unit.api import base
from mistral.tests.unit import base as unit_base
from mistral import utils
from mistral.utils import rest_utils
from mistral.workflow import states
from mistral_lib import utils
# This line is needed for correct initialization of messaging config.
oslo_messaging.get_rpc_transport(cfg.CONF)

View File

@ -25,7 +25,7 @@ from mistral.db.v2.sqlalchemy import models
from mistral import exceptions as exc
from mistral.tests.unit.api import base
from mistral.tests.unit import base as unit_base
from mistral import utils
from mistral_lib import utils
WF_DEFINITION = """
---

View File

@ -17,7 +17,7 @@ import datetime
from mistral.db.v2.sqlalchemy import api as db_api
from mistral.tests.unit import base as test_base
from mistral import utils
from mistral_lib import utils
WF_EXEC = {
'id': 'c0f3be41-88b9-4c86-a669-83e77cd0a1b8',

View File

@ -27,8 +27,8 @@ from mistral.db.v2.sqlalchemy import models as db_models
from mistral import exceptions as exc
from mistral.services import security
from mistral.tests.unit import base as test_base
from mistral import utils
from mistral.utils import filter_utils
from mistral_lib import utils
DEFAULT_CTX = test_base.get_context()

View File

@ -24,7 +24,7 @@ from mistral.services import security
from mistral.services import triggers
from mistral.services import workflows
from mistral.tests.unit.engine import base
from mistral import utils
from mistral_lib import utils
WORKFLOW_LIST = """

View File

@ -20,9 +20,9 @@ from mistral.lang.v2 import tasks as tasks_lang
from mistral.services import workflows as wf_service
from mistral.tests.unit import base as test_base
from mistral.tests.unit.engine import base
from mistral import utils
from mistral.workflow import states
from mistral_lib import actions as actions_base
from mistral_lib import utils
# Use the set_default method to set value otherwise in certain test cases

View File

@ -24,10 +24,10 @@ from mistral.services import workbooks as wb_service
from mistral.services import workflows as wf_service
from mistral.tests.unit import base as test_base
from mistral.tests.unit.engine import base
from mistral import utils
from mistral.workflow import data_flow
from mistral.workflow import states
from mistral_lib import actions as actions_base
from mistral_lib import utils
# TODO(nmakhotkin) Need to write more tests.

View File

@ -19,7 +19,7 @@ from mistral.db.v2.sqlalchemy import api as db_api
from mistral import exceptions as exc
from mistral.expressions import jinja_expression as expr
from mistral.tests.unit import base
from mistral import utils
from mistral_lib import utils
DATA = {
"server": {

View File

@ -25,7 +25,7 @@ from mistral.config import cfg
from mistral import exceptions as exc
from mistral.expressions import yaql_expression as expr
from mistral.tests.unit import base
from mistral import utils
from mistral_lib import utils
CONF = cfg.CONF

View File

@ -19,7 +19,7 @@ import yaml
from mistral import exceptions as exc
from mistral.lang import parser as spec_parser
from mistral.tests.unit import base
from mistral import utils
from mistral_lib import utils
class WorkflowSpecValidationTestCase(base.BaseTest):

View File

@ -16,7 +16,7 @@
import copy
from mistral.tests.unit.lang.v2 import base
from mistral import utils
from mistral_lib import utils
class ActionSpecValidationTest(base.WorkbookSpecValidationTestCase):

View File

@ -16,7 +16,7 @@
from mistral.lang.v2 import workflows
from mistral.tests.unit.lang.v2 import base as v2_base
from mistral import utils
from mistral_lib import utils
class TaskSpecValidationTest(v2_base.WorkflowSpecValidationTestCase):

View File

@ -19,7 +19,7 @@ import yaml
from mistral import exceptions as exc
from mistral.tests.unit.lang.v2 import base
from mistral import utils
from mistral_lib import utils
class WorkflowSpecValidationTest(base.WorkflowSpecValidationTestCase):

View File

@ -16,7 +16,7 @@
from mistral import exceptions as exc
from mistral.tests.unit.rpc.kombu import base
from mistral.tests.unit.rpc.kombu import fake_kombu
from mistral import utils
from mistral_lib import utils
import mock
from six import moves

View File

@ -18,7 +18,7 @@ from mistral.db.v2 import api as db_api
from mistral.lang import parser as spec_parser
from mistral.services import actions as action_service
from mistral.tests.unit import base
from mistral import utils
from mistral_lib import utils
# Use the set_default method to set value otherwise in certain test cases

View File

@ -22,7 +22,7 @@ from mistral.services import expiration_policy
from mistral.services.expiration_policy import ExecutionExpirationPolicy
from mistral.tests.unit import base
from mistral.tests.unit.base import get_context
from mistral import utils
from mistral_lib import utils
from oslo_config import cfg

View File

@ -25,7 +25,7 @@ from mistral.services import security
from mistral.services import triggers as t_s
from mistral.services import workflows
from mistral.tests.unit import base
from mistral import utils
from mistral_lib import utils
# Use the set_default method to set value otherwise in certain test cases
# the change in value is not permanent.

View File

@ -24,8 +24,8 @@ from mistral.lang.v2 import tasks
from mistral.lang.v2 import workflows
from mistral.services import workflows as wf_service
from mistral.tests.unit import base
from mistral import utils
from mistral.workflow import states
from mistral_lib import utils
# Use the set_default method to set value otherwise in certain test cases

View File

@ -125,7 +125,8 @@ class ServiceTest(base.BaseTest):
# new coordination configuration.
coordination.cleanup_service_coordinator()
@mock.patch('mistral.utils.get_process_identifier', return_value='fake_id')
@mock.patch('mistral_lib.utils.get_process_identifier',
return_value='fake_id')
def test_register_membership(self, mock_get_identifier):
cfg.CONF.set_default('backend_url', 'zake://', 'coordination')

View File

@ -14,75 +14,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import testtools.matchers as ttm
from mistral import exceptions as exc
from mistral.tests.unit import base
from mistral import utils
from mistral.utils import ssh_utils
LEFT = {
'key1': {
'key11': "val11"
},
'key2': 'val2'
}
RIGHT = {
'key1': {
'key11': "val111111",
'key12': "val12",
'key13': {
'key131': 'val131'
}
},
'key2': 'val2222',
'key3': 'val3'
}
from mistral_lib import utils
class UtilsTest(base.BaseTest):
def test_merge_dicts(self):
left = copy.deepcopy(LEFT)
right = copy.deepcopy(RIGHT)
expected = {
'key1': {
'key11': "val111111",
'key12': "val12",
'key13': {
'key131': 'val131'
}
},
'key2': 'val2222',
'key3': 'val3'
}
utils.merge_dicts(left, right)
self.assertDictEqual(left, expected)
def test_merge_dicts_overwrite_false(self):
left = copy.deepcopy(LEFT)
right = copy.deepcopy(RIGHT)
expected = {
'key1': {
'key11': "val11",
'key12': "val12",
'key13': {
'key131': 'val131'
}
},
'key2': 'val2',
'key3': 'val3'
}
utils.merge_dicts(left, right, overwrite=False)
self.assertDictEqual(left, expected)
def test_itersubclasses(self):
class A(object):
@ -99,27 +37,6 @@ class UtilsTest(base.BaseTest):
self.assertEqual([B, C, D], list(utils.iter_subclasses(A)))
def test_get_dict_from_entries(self):
input = ['param1', {'param2': 2}]
input_dict = utils.get_dict_from_entries(input)
self.assertIn('param1', input_dict)
self.assertIn('param2', input_dict)
self.assertEqual(2, input_dict.get('param2'))
self.assertIs(input_dict.get('param1'), utils.NotDefined)
def test_get_input_dict_from_string(self):
self.assertDictEqual(
{
'param1': utils.NotDefined,
'param2': 2,
'param3': 'var3'
},
utils.get_dict_from_string('param1, param2=2, param3="var3"')
)
self.assertDictEqual({}, utils.get_dict_from_string(''))
def test_paramiko_to_private_key(self):
self.assertRaises(
exc.DataAccessException,
@ -131,133 +48,3 @@ class UtilsTest(base.BaseTest):
ssh_utils._to_paramiko_private_key,
"..\\dir"
)
def test_cut_string(self):
s = 'Hello, Mistral!'
self.assertEqual('Hello...', utils.cut_string(s, length=5))
self.assertEqual(s, utils.cut_string(s, length=100))
self.assertEqual(s, utils.cut_string(s, length=-1))
def test_cut_list(self):
l = ['Hello, Mistral!', 'Hello, OpenStack!']
self.assertEqual("['Hello, M...", utils.cut_list(l, 13))
self.assertEqual("['Hello, Mistr...", utils.cut_list(l, 17))
self.assertEqual("['Hello, Mistral!', 'He...", utils.cut_list(l, 26))
self.assertEqual(
"['Hello, Mistral!', 'Hello, OpenStack!']",
utils.cut_list(l, 100)
)
self.assertEqual(
"['Hello, Mistral!', 'Hello, OpenStack!']",
utils.cut_list(l, -1)
)
self.assertEqual("[1, 2...", utils.cut_list([1, 2, 3, 4, 5], 8))
self.assertEqual("[1, 2,...", utils.cut_list([1, 2, 3, 4, 5], 9))
self.assertEqual("[1, 2, 3...", utils.cut_list([1, 2, 3, 4, 5], 11))
self.assertRaises(ValueError, utils.cut_list, (1, 2))
def test_cut_list_with_large_dict_of_str(self):
d = [str(i) for i in range(65535)]
s = utils.cut(d, 65535)
self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
def test_cut_list_with_large_dict_of_int(self):
d = [i for i in range(65535)]
s = utils.cut(d, 65535)
self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
def test_cut_list_with_large_dict_of_dict(self):
d = [{'value': str(i)} for i in range(65535)]
s = utils.cut(d, 65535)
self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
def test_cut_list_for_state_info(self):
d = [{'value': 'This is a string that exceeds 35 characters'}
for i in range(2000)]
s = utils.cut(d, 65500)
self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65500)))
def test_cut_dict_with_strings(self):
d = {'key1': 'value1', 'key2': 'value2'}
s = utils.cut_dict(d, 13)
self.assertIn(s, ["{'key1': '...", "{'key2': '..."])
s = utils.cut_dict(d, 15)
self.assertIn(s, ["{'key1': 'va...", "{'key2': 'va..."])
s = utils.cut_dict(d, 22)
self.assertIn(
s,
["{'key1': 'value1', ...", "{'key2': 'value2', ..."]
)
self.assertIn(
utils.cut_dict(d, 100),
[
"{'key1': 'value1', 'key2': 'value2'}",
"{'key2': 'value2', 'key1': 'value1'}"
]
)
self.assertIn(
utils.cut_dict(d, -1),
[
"{'key1': 'value1', 'key2': 'value2'}",
"{'key2': 'value2', 'key1': 'value1'}"
]
)
self.assertRaises(ValueError, utils.cut_dict, (1, 2))
def test_cut_dict_with_digits(self):
d = {1: 2, 3: 4}
s = utils.cut_dict(d, 10)
self.assertIn(s, ["{1: 2, ...", "{3: 4, ..."])
s = utils.cut_dict(d, 11)
self.assertIn(s, ["{1: 2, 3...", "{3: 4, 1..."])
s = utils.cut_dict(d, 100)
self.assertIn(s, ["{1: 2, 3: 4}", "{3: 4, 1: 2}"])
def test_cut_dict_with_large_dict_of_str(self):
d = {}
for i in range(65535):
d[str(i)] = str(i)
s = utils.cut(d, 65535)
self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
def test_cut_dict_with_large_dict_of_int(self):
d = {}
for i in range(65535):
d[i] = i
s = utils.cut(d, 65535)
self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
def test_cut_dict_with_large_dict_of_dict(self):
d = {}
for i in range(65535):
d[i] = {'value': str(i)}
s = utils.cut(d, 65535)
self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
def test_cut_dict_for_state_info(self):
d = {}
for i in range(2000):
d[i] = {'value': 'This is a string that exceeds 35 characters'}
s = utils.cut(d, 65500)
self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65500)))

View File

@ -15,30 +15,15 @@
# limitations under the License.
import contextlib
import datetime
import functools
import json
import os
from os import path
import shutil
import socket
import string
import sys
import tempfile
import threading
import eventlet
from eventlet import corolocal
from oslo_concurrency import processutils
from oslo_log import log as logging
from oslo_utils import timeutils
from oslo_utils import uuidutils
import pkg_resources as pkg
import random
from mistral import exceptions as exc
# Thread local storage.
_th_loc_storage = threading.local()
@ -49,380 +34,6 @@ ACTION_TASK_TYPE = 'ACTION'
WORKFLOW_TASK_TYPE = 'WORKFLOW'
def generate_unicode_uuid():
return uuidutils.generate_uuid()
def is_valid_uuid(uuid_string):
return uuidutils.is_uuid_like(uuid_string)
def _get_greenlet_local_storage():
greenlet_id = corolocal.get_ident()
greenlet_locals = getattr(_th_loc_storage, "greenlet_locals", None)
if not greenlet_locals:
greenlet_locals = {}
_th_loc_storage.greenlet_locals = greenlet_locals
if greenlet_id in greenlet_locals:
return greenlet_locals[greenlet_id]
else:
return None
def has_thread_local(var_name):
gl_storage = _get_greenlet_local_storage()
return gl_storage and var_name in gl_storage
def get_thread_local(var_name):
if not has_thread_local(var_name):
return None
return _get_greenlet_local_storage()[var_name]
def set_thread_local(var_name, val):
if val is None and has_thread_local(var_name):
gl_storage = _get_greenlet_local_storage()
# Delete variable from greenlet local storage.
if gl_storage:
del gl_storage[var_name]
# Delete the entire greenlet local storage from thread local storage.
if gl_storage and len(gl_storage) == 0:
del _th_loc_storage.greenlet_locals[corolocal.get_ident()]
if val is not None:
gl_storage = _get_greenlet_local_storage()
if not gl_storage:
gl_storage = _th_loc_storage.greenlet_locals[
corolocal.get_ident()] = {}
gl_storage[var_name] = val
def log_exec(logger, level=logging.DEBUG):
"""Decorator for logging function execution.
By default, target function execution is logged with DEBUG level.
"""
def _decorator(func):
@functools.wraps(func)
def _logged(*args, **kw):
params_repr = ("[args=%s, kw=%s]" % (str(args), str(kw))
if args or kw else "")
func_repr = ("Called method [name=%s, doc='%s', params=%s]" %
(func.__name__, func.__doc__, params_repr))
logger.log(level, func_repr)
return func(*args, **kw)
_logged.__doc__ = func.__doc__
return _logged
return _decorator
def merge_dicts(left, right, overwrite=True):
"""Merges two dictionaries.
Values of right dictionary recursively get merged into left dictionary.
:param left: Left dictionary.
:param right: Right dictionary.
:param overwrite: If False, left value will not be overwritten if exists.
"""
if left is None:
return right
if right is None:
return left
for k, v in right.items():
if k not in left:
left[k] = v
else:
left_v = left[k]
if isinstance(left_v, dict) and isinstance(v, dict):
merge_dicts(left_v, v, overwrite=overwrite)
elif overwrite:
left[k] = v
return left
def update_dict(left, right):
"""Updates left dict with content from right dict
:param left: Left dict.
:param right: Right dict.
:return: the updated left dictionary.
"""
if left is None:
return right
if right is None:
return left
left.update(right)
return left
def get_file_list(directory):
base_path = pkg.resource_filename("mistral", directory)
return [path.join(base_path, f) for f in os.listdir(base_path)
if path.isfile(path.join(base_path, f))]
def cut_dict(d, length=100):
"""Removes dictionary entries according to the given length.
This method removes a number of entries, if needed, so that a
string representation would fit into the given length.
The intention of this method is to optimize truncation of string
representation for dictionaries where the exact precision is not
critically important. Otherwise, we'd always have to convert a dict
into a string first and then shrink it to a needed size which will
increase memory footprint and reduce performance in case of large
dictionaries (i.e. tens of thousands entries).
Note that the method, due to complexity of the algorithm, has some
non-zero precision which depends on exact keys and values placed into
the dict. So for some dicts their reduced string representations will
be only approximately equal to the given value (up to around several
chars difference).
:param d: A dictionary.
:param length: A length limiting the dictionary string representation.
:return: A dictionary which is a subset of the given dictionary.
"""
if not isinstance(d, dict):
raise ValueError("A dictionary is expected, got: %s" % type(d))
res = "{"
idx = 0
for key, value in d.items():
k = str(key)
v = str(value)
# Processing key.
new_len = len(k)
is_str = isinstance(key, str)
if is_str:
new_len += 2 # Account for the quotation marks
if 0 <= length <= new_len + len(res):
res += "'%s" % k if is_str else k
break
else:
res += "'%s'" % k if is_str else k
res += ": "
# Processing value.
new_len = len(v)
is_str = isinstance(value, str)
if is_str:
new_len += 2
if 0 <= length <= new_len + len(res):
res += "'%s" % v if is_str else v
break
else:
res += "'%s'" % v if is_str else v
res += ', ' if idx < len(d) - 1 else '}'
idx += 1
if 0 <= length <= len(res) and res[length - 1] is not '}':
res = res[:length - 3] + '...'
return res
def cut_list(l, length=100):
if not isinstance(l, list):
raise ValueError("A list is expected, got: %s" % type(l))
res = '['
for idx, item in enumerate(l):
s = str(item)
new_len = len(res) + len(s)
is_str = isinstance(item, str)
if is_str:
new_len += 2
if 0 <= length <= new_len:
res += "'%s" % s if is_str else s
break
else:
res += "'%s'" % s if is_str else s
res += ', ' if idx < len(l) - 1 else ']'
if 0 <= length <= len(res) and res[length - 1] is not ']':
res = res[:length - 3] + '...'
return res
def cut_string(s, length=100):
if 0 <= length < len(s):
return "%s..." % s[:length]
return s
def cut(data, length=100):
if not data:
return data
if isinstance(data, list):
return cut_list(data, length=length)
if isinstance(data, dict):
return cut_dict(data, length=length)
return cut_string(str(data), length=length)
def cut_by_kb(data, kilobytes):
length = get_number_of_chars_from_kilobytes(kilobytes)
return cut(data, length)
def cut_by_char(data, length):
return cut(data, length)
def iter_subclasses(cls, _seen=None):
"""Generator over all subclasses of a given class in depth first order."""
if not isinstance(cls, type):
raise TypeError('iter_subclasses must be called with new-style class'
', not %.100r' % cls)
_seen = _seen or set()
try:
subs = cls.__subclasses__()
except TypeError: # fails only when cls is type
subs = cls.__subclasses__(cls)
for sub in subs:
if sub not in _seen:
_seen.add(sub)
yield sub
for _sub in iter_subclasses(sub, _seen):
yield _sub
def random_sleep(limit=1):
"""Sleeps for a random period of time not exceeding the given limit.
Mostly intended to be used by tests to emulate race conditions.
:param limit: Float number of seconds that a sleep period must not exceed.
"""
seconds = random.Random().randint(0, limit * 1000) * 0.001
print("Sleep: %s sec..." % seconds)
eventlet.sleep(seconds)
class NotDefined(object):
"""Marker of an empty value.
In a number of cases None can't be used to express the semantics of
a not defined value because None is just a normal value rather than
a value set to denote that it's not defined. This class can be used
in such cases instead of None.
"""
pass
def get_number_of_chars_from_kilobytes(kilobytes):
bytes_per_char = sys.getsizeof('s') - sys.getsizeof('')
total_number_of_chars = int(kilobytes * 1024 / bytes_per_char)
return total_number_of_chars
def get_dict_from_string(string, delimiter=','):
if not string:
return {}
kv_dicts = []
for kv_pair_str in string.split(delimiter):
kv_str = kv_pair_str.strip()
kv_list = kv_str.split('=')
if len(kv_list) > 1:
try:
value = json.loads(kv_list[1])
except ValueError:
value = kv_list[1]
kv_dicts += [{kv_list[0]: value}]
else:
kv_dicts += [kv_list[0]]
return get_dict_from_entries(kv_dicts)
def get_dict_from_entries(entries):
"""Transforms a list of entries into dictionary.
:param entries: A list of entries.
If an entry is a dictionary the method simply updates the result
dictionary with its content.
If an entry is not a dict adds {entry, NotDefined} into the result.
"""
result = {}
for e in entries:
if isinstance(e, dict):
result.update(e)
else:
# NOTE(kong): we put NotDefined here as the value of
# param without value specified, to distinguish from
# the valid values such as None, ''(empty string), etc.
result[e] = NotDefined
return result
def get_process_identifier():
"""Gets current running process identifier."""
return "%s_%s" % (socket.gethostname(), os.getpid())
@contextlib.contextmanager
def tempdir(**kwargs):
argdict = kwargs.copy()
@ -490,55 +101,3 @@ def generate_key_pair(key_length=2048):
public_key = open(public_key_path).read()
return private_key, public_key
def utc_now_sec():
"""Returns current time and drops microseconds."""
return drop_microseconds(timeutils.utcnow())
def drop_microseconds(date):
"""Drops microseconds and returns date."""
return date.replace(microsecond=0)
def datetime_to_str(val, sep=' '):
"""Converts datetime value to string.
If the given value is not an instance of datetime then the method
returns the same value.
:param val: datetime value.
:param sep: Separator between date and time.
:return: Datetime as a string.
"""
if isinstance(val, datetime.datetime):
return val.isoformat(sep)
return val
def datetime_to_str_in_dict(d, key, sep=' '):
"""Converts datetime value in te given dict to string.
:param d: A dictionary.
:param key: The key for which we need to convert the value.
:param sep: Separator between date and time.
"""
val = d.get(key)
if val is not None:
d[key] = datetime_to_str(d[key], sep=sep)
def generate_string(length):
"""Returns random string.
:param length: the length of returned string
"""
return ''.join(random.choice(
string.ascii_uppercase + string.digits)
for _ in range(length)
)

View File

@ -27,8 +27,8 @@ from yaql.language import utils as yaql_utils
from mistral.config import cfg
from mistral.db.v2 import api as db_api
from mistral import utils
from mistral.utils import filter_utils
from mistral_lib import utils
# TODO(rakhmerov): it's work around the bug in YAQL.
# YAQL shouldn't expose internal types to custom functions.

View File

@ -21,7 +21,7 @@ from oslo_log import log as logging
import osprofiler.profiler
import osprofiler.web
from mistral import utils
from mistral_lib import utils
PROFILER_LOG = logging.getLogger(cfg.CONF.profiler.profiler_log_name)

View File

@ -23,10 +23,10 @@ from osprofiler import profiler
from mistral.db.v2 import api as db_api
from mistral import exceptions as exc
from mistral.lang import parser as spec_parser
from mistral import utils as u
from mistral.workflow import commands
from mistral.workflow import data_flow
from mistral.workflow import states
from mistral_lib import utils as u
LOG = logging.getLogger(__name__)

View File

@ -21,9 +21,9 @@ from mistral.db.v2.sqlalchemy import models
from mistral import exceptions as exc
from mistral import expressions as expr
from mistral.lang import parser as spec_parser
from mistral import utils
from mistral.utils import inspect_utils
from mistral.workflow import states
from mistral_lib import utils
LOG = logging.getLogger(__name__)
CONF = cfg.CONF

View File

@ -18,11 +18,11 @@ from osprofiler import profiler
from mistral.db.v2 import api as db_api
from mistral import exceptions as exc
from mistral import expressions as expr
from mistral import utils
from mistral.workflow import base
from mistral.workflow import commands
from mistral.workflow import data_flow
from mistral.workflow import states
from mistral_lib import utils
LOG = logging.getLogger(__name__)

View File

@ -13,7 +13,7 @@ gnocchiclient>=3.3.1 # Apache-2.0
Jinja2>=2.10 # BSD License (3 clause)
#jsonschema>=2.6.0 # MIT
keystonemiddleware>=4.18.0 # Apache-2.0
mistral-lib>=0.4.0 # Apache-2.0
mistral-lib>=1.2.0 # Apache-2.0
networkx<2.3,>=1.10;python_version<'3.0' # BSD
networkx>=2.3;python_version>='3.4' # BSD
oslo.concurrency>=3.26.0 # Apache-2.0

View File

@ -8,7 +8,7 @@ croniter>=0.3.4 # MIT License
doc8>=0.6.0 # Apache-2.0
fixtures>=3.0.0 # Apache-2.0/BSD
keystonemiddleware>=4.18.0 # Apache-2.0
mistral-lib>=0.4.0 # Apache-2.0
mistral-lib>=1.2.0 # Apache-2.0
mock>=2.0.0 # BSD
networkx<2.3,>=1.10;python_version<'3.0' # BSD
networkx>=2.3;python_version>='3.4' # BSD