Merge "moved generic util functions from mistral to mistral-lib"

lower-constraints.txt

@@ -50,7 +50,7 @@ logutils==0.3.5
 Mako==0.4.0
 MarkupSafe==1.0
 mccabe==0.2.1
-mistral-lib==0.4.0
+mistral-lib==1.2.0
 mock==2.0.0
 monotonic==0.6
 mox3==0.20.0

mistral/api/controllers/resource.py

@@ -16,7 +16,7 @@ import json
 
 from wsme import types as wtypes
 
-from mistral import utils
+from mistral_lib import utils
 
 
 class Resource(wtypes.Base):

mistral/api/controllers/v2/environment.py

@@ -25,9 +25,9 @@ from mistral.api.controllers.v2 import types
 from mistral import context
 from mistral.db.v2 import api as db_api
 from mistral import exceptions
-from mistral.utils import cut
 from mistral.utils import filter_utils
 from mistral.utils import rest_utils
+from mistral_lib.utils import cut
 
 
 LOG = logging.getLogger(__name__)

mistral/api/controllers/v2/execution.py

@@ -34,9 +34,9 @@ from mistral import exceptions as exc
 from mistral.rpc import clients as rpc
 from mistral.services import workflows as wf_service
 from mistral.utils import filter_utils
-from mistral.utils import merge_dicts
 from mistral.utils import rest_utils
 from mistral.workflow import states
+from mistral_lib.utils import merge_dicts
 
 LOG = logging.getLogger(__name__)
 

mistral/api/controllers/v2/resources.py

@@ -19,8 +19,8 @@ from wsme import types as wtypes
 from mistral.api.controllers import resource
 from mistral.api.controllers.v2 import types
 from mistral import exceptions as exc
-from mistral import utils
 from mistral.workflow import states
+from mistral_lib import utils
 
 SCOPE_TYPES = wtypes.Enum(str, 'private', 'public')
 

mistral/context.py

@@ -27,7 +27,7 @@ from pecan import hooks
 
 from mistral import auth
 from mistral import exceptions as exc
-from mistral import utils
+from mistral_lib import utils
 
 CONF = cfg.CONF
 _CTX_THREAD_LOCAL_NAME = "MISTRAL_APP_CTX_THREAD_LOCAL"

mistral/db/sqlalchemy/base.py

@@ -22,7 +22,7 @@ import sqlalchemy as sa
 
 from mistral.db.sqlalchemy import sqlite_lock
 from mistral import exceptions as exc
-from mistral import utils
+from mistral_lib import utils
 
 
 # Note(dzimine): sqlite only works for basic testing.

mistral/db/sqlalchemy/migration/alembic_migrations/versions/027_add_last_heartbeat_to_action_execution.py

@@ -27,7 +27,7 @@ down_revision = '026'
 
 from alembic import op
 import datetime
-from mistral import utils
+from mistral_lib import utils
 from oslo_config import cfg
 from sqlalchemy import Column, DateTime, Boolean
 

mistral/db/sqlalchemy/model_base.py

@@ -19,7 +19,7 @@ from sqlalchemy.ext import declarative
 from sqlalchemy.orm import attributes
 
 from mistral.services import security
-from mistral import utils
+from mistral_lib import utils
 
 
 def id_column():

mistral/db/v2/sqlalchemy/api.py

@@ -38,8 +38,8 @@ from mistral.db.v2.sqlalchemy import filters as db_filters
 from mistral.db.v2.sqlalchemy import models
 from mistral import exceptions as exc
 from mistral.services import security
-from mistral import utils
 from mistral.workflow import states
+from mistral_lib import utils
 
 
 CONF = cfg.CONF

mistral/db/v2/sqlalchemy/models.py

@@ -29,7 +29,7 @@ from mistral.db.sqlalchemy import model_base as mb
 from mistral.db.sqlalchemy import types as st
 from mistral import exceptions as exc
 from mistral.services import security
-from mistral import utils
+from mistral_lib import utils
 
 
 # Definition objects.

mistral/engine/actions.py

@@ -31,11 +31,11 @@ from mistral.lang import parser as spec_parser
 from mistral.rpc import clients as rpc
 from mistral.services import action_manager as a_m
 from mistral.services import security
-from mistral import utils
 from mistral.utils import wf_trace
 from mistral.workflow import data_flow
 from mistral.workflow import states
 from mistral_lib import actions as ml_actions
+from mistral_lib import utils
 
 
 LOG = logging.getLogger(__name__)

mistral/engine/default_engine.py

@@ -29,8 +29,8 @@ from mistral.engine import base
 from mistral.engine import post_tx_queue
 from mistral.engine import workflow_handler as wf_handler
 from mistral import exceptions
-from mistral import utils as u
 from mistral.workflow import states
+from mistral_lib import utils as u
 
 
 # Submodules of mistral.engine will throw NoSuchOptError if configuration

mistral/engine/engine_server.py

@@ -22,8 +22,8 @@ from mistral.scheduler import base as sched_base
 from mistral.service import base as service_base
 from mistral.services import action_execution_checker
 from mistral.services import expiration_policy
-from mistral import utils
 from mistral.utils import profiler as profiler_utils
+from mistral_lib import utils
 
 LOG = logging.getLogger(__name__)
 

mistral/engine/post_tx_queue.py

@@ -20,7 +20,7 @@ from oslo_log import log as logging
 from mistral import context
 from mistral.db import utils as db_utils
 from mistral.db.v2 import api as db_api
-from mistral import utils
+from mistral_lib import utils
 
 
 """

mistral/engine/tasks.py

@@ -32,12 +32,12 @@ from mistral import exceptions as exc
 from mistral import expressions as expr
 from mistral.notifiers import base as notif
 from mistral.notifiers import notification_events as events
-from mistral import utils
 from mistral.utils import wf_trace
 from mistral.workflow import base as wf_base
 from mistral.workflow import commands
 from mistral.workflow import data_flow
 from mistral.workflow import states
+from mistral_lib import utils
 
 
 LOG = logging.getLogger(__name__)

mistral/engine/utils.py

@@ -17,7 +17,7 @@ import copy
 
 from mistral.db.v2 import api as db_api
 from mistral import exceptions as exc
-from mistral import utils
+from mistral_lib import utils
 
 
 def _compare_parameters(expected_input, actual_input):

mistral/engine/workflows.py

@@ -34,14 +34,13 @@ from mistral.notifiers import notification_events as events
 from mistral.rpc import clients as rpc
 from mistral.services import triggers
 from mistral.services import workflows as wf_service
-from mistral import utils
-from mistral.utils import merge_dicts
 from mistral.utils import wf_trace
 from mistral.workflow import base as wf_base
 from mistral.workflow import commands
 from mistral.workflow import data_flow
 from mistral.workflow import states
 from mistral_lib import actions as ml_actions
+from mistral_lib import utils
 
 
 LOG = logging.getLogger(__name__)
@@ -538,7 +537,7 @@ class Workflow(object):
                 cfg.CONF.engine.execution_field_size_limit_kb
             )
 
-        self.wf_ex.output = merge_dicts({'result': msg}, output_on_error)
+        self.wf_ex.output = utils.merge_dicts({'result': msg}, output_on_error)
 
         # Publish event.
         self.notify(events.WORKFLOW_FAILED)

mistral/executors/executor_server.py

@@ -19,8 +19,8 @@ from mistral.executors import default_executor as exe
 from mistral.rpc import base as rpc
 from mistral.service import base as service_base
 from mistral.services import action_execution_reporter
-from mistral import utils
 from mistral.utils import profiler as profiler_utils
+from mistral_lib import utils
 
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)

mistral/expressions/yaql_expression.py

@@ -26,8 +26,8 @@ from yaql.language import factory
 from mistral.config import cfg
 from mistral import exceptions as exc
 from mistral.expressions.base_expression import Evaluator
-from mistral import utils
 from mistral.utils import expression_utils
+from mistral_lib import utils
 
 LOG = logging.getLogger(__name__)
 

mistral/lang/base.py

@@ -24,7 +24,7 @@ from mistral import expressions as expr
 from mistral.expressions.jinja_expression import ANY_JINJA_REGEXP
 from mistral.expressions.yaql_expression import INLINE_YAQL_REGEXP
 from mistral.lang import types
-from mistral import utils
+from mistral_lib import utils
 
 ACTION_PATTERNS = {
     "command": r"[\w\.]+[^=\(\s\"]*",

mistral/lang/v2/actions.py

@@ -17,7 +17,7 @@ import six
 
 from mistral.lang import types
 from mistral.lang.v2 import base
-from mistral import utils
+from mistral_lib import utils
 
 
 class ActionSpec(base.BaseSpec):

mistral/lang/v2/tasks.py

@@ -26,8 +26,8 @@ from mistral.lang.v2 import on_clause
 from mistral.lang.v2 import policies
 from mistral.lang.v2 import publish
 from mistral.lang.v2 import retry_policy
-from mistral import utils
 from mistral.workflow import states
+from mistral_lib import utils
 
 _expr_ptrns = [expressions.patterns[name] for name in expressions.patterns]
 WITH_ITEMS_PTRN = re.compile(

mistral/lang/v2/workflows.py

@@ -21,7 +21,7 @@ from mistral.lang import types
 from mistral.lang.v2 import base
 from mistral.lang.v2 import task_defaults
 from mistral.lang.v2 import tasks
-from mistral import utils
+from mistral_lib import utils
 
 NOOP_COMMAND = 'noop'
 FAIL_COMMAND = 'fail'

mistral/notifiers/notification_server.py

@@ -18,8 +18,8 @@ from mistral import config as cfg
 from mistral.notifiers import default_notifier as notif
 from mistral.rpc import base as rpc
 from mistral.service import base as service_base
-from mistral import utils
 from mistral.utils import profiler as profiler_utils
+from mistral_lib import utils
 
 LOG = logging.getLogger(__name__)
 

mistral/rpc/kombu/kombu_client.py

@@ -28,7 +28,7 @@ from mistral.rpc import base as rpc_base
 from mistral.rpc.kombu import base as kombu_base
 from mistral.rpc.kombu import kombu_hosts
 from mistral.rpc.kombu import kombu_listener
-from mistral import utils
+from mistral_lib import utils
 
 #: When connection to the RabbitMQ server breaks, the
 #: client will receive EPIPE socket errors. These indicate

mistral/scheduler/default_scheduler.py

@@ -28,7 +28,7 @@ from mistral.db import utils as db_utils
 from mistral.db.v2 import api as db_api
 from mistral import exceptions as exc
 from mistral.scheduler import base
-from mistral import utils
+from mistral_lib import utils
 
 
 LOG = logging.getLogger(__name__)

mistral/service/coordination.py

@@ -21,7 +21,7 @@ from oslo_service import threadgroup
 import tenacity
 import tooz.coordination
 
-from mistral import utils
+from mistral_lib import utils
 
 
 LOG = log.getLogger(__name__)

mistral/services/action_execution_checker.py

@@ -21,8 +21,8 @@ from mistral.db import utils as db_utils
 from mistral.db.v2 import api as db_api
 from mistral.engine import action_handler
 from mistral.engine import post_tx_queue
-from mistral import utils
 from mistral_lib import actions as mistral_lib
+from mistral_lib import utils
 from oslo_config import cfg
 from oslo_log import log as logging
 

mistral/services/action_manager.py

@@ -21,8 +21,8 @@ from mistral.actions import generator_factory
 from mistral.db.v2 import api as db_api
 from mistral import exceptions as exc
 from mistral.services import actions
-from mistral import utils
 from mistral.utils import inspect_utils as i_utils
+from mistral_lib import utils
 
 
 # TODO(rakhmerov): Make methods more consistent and granular.

mistral/services/legacy_scheduler.py

@@ -29,7 +29,7 @@ from mistral.db import utils as db_utils
 from mistral.db.v2 import api as db_api
 from mistral import exceptions as exc
 from mistral.scheduler import base as sched_base
-from mistral import utils
+from mistral_lib import utils
 
 
 LOG = logging.getLogger(__name__)

mistral/services/workflows.py

@@ -16,8 +16,8 @@ import yaml
 from mistral.db.v2 import api as db_api
 from mistral import exceptions as exc
 from mistral.lang import parser as spec_parser
-from mistral import utils
 from mistral.workflow import states
+from mistral_lib import utils
 from oslo_log import log as logging
 
 

mistral/tests/unit/api/test_resource_base.py

@@ -18,7 +18,7 @@ import datetime
 from mistral.api.controllers.v2 import resources
 from mistral.db.v2 import api as db_api
 from mistral.tests.unit import base
-from mistral import utils
+from mistral_lib import utils
 
 
 WF_EXEC = {

mistral/tests/unit/api/v2/test_actions.py

@@ -21,7 +21,7 @@ from mistral.db.v2 import api as db_api
 from mistral.db.v2.sqlalchemy import models
 from mistral import exceptions as exc
 from mistral.tests.unit.api import base
-from mistral import utils
+from mistral_lib import utils
 
 
 ACTION_DEFINITION = """

mistral/tests/unit/api/v2/test_executions.py

@@ -36,9 +36,9 @@ from mistral.rpc import base as rpc_base
 from mistral.rpc import clients as rpc_clients
 from mistral.tests.unit.api import base
 from mistral.tests.unit import base as unit_base
-from mistral import utils
 from mistral.utils import rest_utils
 from mistral.workflow import states
+from mistral_lib import utils
 
 # This line is needed for correct initialization of messaging config.
 oslo_messaging.get_rpc_transport(cfg.CONF)

mistral/tests/unit/api/v2/test_workflows.py

@@ -25,7 +25,7 @@ from mistral.db.v2.sqlalchemy import models
 from mistral import exceptions as exc
 from mistral.tests.unit.api import base
 from mistral.tests.unit import base as unit_base
-from mistral import utils
+from mistral_lib import utils
 
 WF_DEFINITION = """
 ---

mistral/tests/unit/db/v2/test_db_model.py

@@ -17,7 +17,7 @@ import datetime
 
 from mistral.db.v2.sqlalchemy import api as db_api
 from mistral.tests.unit import base as test_base
-from mistral import utils
+from mistral_lib import utils
 
 WF_EXEC = {
     'id': 'c0f3be41-88b9-4c86-a669-83e77cd0a1b8',

mistral/tests/unit/db/v2/test_sqlalchemy_db_api.py

@@ -27,8 +27,8 @@ from mistral.db.v2.sqlalchemy import models as db_models
 from mistral import exceptions as exc
 from mistral.services import security
 from mistral.tests.unit import base as test_base
-from mistral import utils
 from mistral.utils import filter_utils
+from mistral_lib import utils
 
 
 DEFAULT_CTX = test_base.get_context()

mistral/tests/unit/engine/test_cron_trigger.py

@@ -24,7 +24,7 @@ from mistral.services import security
 from mistral.services import triggers
 from mistral.services import workflows
 from mistral.tests.unit.engine import base
-from mistral import utils
+from mistral_lib import utils
 
 
 WORKFLOW_LIST = """

mistral/tests/unit/engine/test_join.py

@@ -20,9 +20,9 @@ from mistral.lang.v2 import tasks as tasks_lang
 from mistral.services import workflows as wf_service
 from mistral.tests.unit import base as test_base
 from mistral.tests.unit.engine import base
-from mistral import utils
 from mistral.workflow import states
 from mistral_lib import actions as actions_base
+from mistral_lib import utils
 
 
 # Use the set_default method to set value otherwise in certain test cases

mistral/tests/unit/engine/test_with_items.py

@@ -24,10 +24,10 @@ from mistral.services import workbooks as wb_service
 from mistral.services import workflows as wf_service
 from mistral.tests.unit import base as test_base
 from mistral.tests.unit.engine import base
-from mistral import utils
 from mistral.workflow import data_flow
 from mistral.workflow import states
 from mistral_lib import actions as actions_base
+from mistral_lib import utils
 
 # TODO(nmakhotkin) Need to write more tests.
 

mistral/tests/unit/expressions/test_jinja_expression.py

@@ -19,7 +19,7 @@ from mistral.db.v2.sqlalchemy import api as db_api
 from mistral import exceptions as exc
 from mistral.expressions import jinja_expression as expr
 from mistral.tests.unit import base
-from mistral import utils
+from mistral_lib import utils
 
 DATA = {
     "server": {

mistral/tests/unit/expressions/test_yaql_expression.py

@@ -25,7 +25,7 @@ from mistral.config import cfg
 from mistral import exceptions as exc
 from mistral.expressions import yaql_expression as expr
 from mistral.tests.unit import base
-from mistral import utils
+from mistral_lib import utils
 
 CONF = cfg.CONF
 

mistral/tests/unit/lang/v2/base.py

@@ -19,7 +19,7 @@ import yaml
 from mistral import exceptions as exc
 from mistral.lang import parser as spec_parser
 from mistral.tests.unit import base
-from mistral import utils
+from mistral_lib import utils
 
 
 class WorkflowSpecValidationTestCase(base.BaseTest):

mistral/tests/unit/lang/v2/test_actions.py

@@ -16,7 +16,7 @@
 import copy
 
 from mistral.tests.unit.lang.v2 import base
-from mistral import utils
+from mistral_lib import utils
 
 
 class ActionSpecValidationTest(base.WorkbookSpecValidationTestCase):

mistral/tests/unit/lang/v2/test_tasks.py

@@ -16,7 +16,7 @@
 
 from mistral.lang.v2 import workflows
 from mistral.tests.unit.lang.v2 import base as v2_base
-from mistral import utils
+from mistral_lib import utils
 
 
 class TaskSpecValidationTest(v2_base.WorkflowSpecValidationTestCase):

mistral/tests/unit/lang/v2/test_workflows.py

@@ -19,7 +19,7 @@ import yaml
 
 from mistral import exceptions as exc
 from mistral.tests.unit.lang.v2 import base
-from mistral import utils
+from mistral_lib import utils
 
 
 class WorkflowSpecValidationTest(base.WorkflowSpecValidationTestCase):

mistral/tests/unit/rpc/kombu/test_kombu_listener.py

@@ -16,7 +16,7 @@
 from mistral import exceptions as exc
 from mistral.tests.unit.rpc.kombu import base
 from mistral.tests.unit.rpc.kombu import fake_kombu
-from mistral import utils
+from mistral_lib import utils
 
 import mock
 from six import moves

mistral/tests/unit/services/test_action_service.py

@@ -18,7 +18,7 @@ from mistral.db.v2 import api as db_api
 from mistral.lang import parser as spec_parser
 from mistral.services import actions as action_service
 from mistral.tests.unit import base
-from mistral import utils
+from mistral_lib import utils
 
 
 # Use the set_default method to set value otherwise in certain test cases

mistral/tests/unit/services/test_expiration_policy.py

@@ -22,7 +22,7 @@ from mistral.services import expiration_policy
 from mistral.services.expiration_policy import ExecutionExpirationPolicy
 from mistral.tests.unit import base
 from mistral.tests.unit.base import get_context
-from mistral import utils
+from mistral_lib import utils
 from oslo_config import cfg
 
 

mistral/tests/unit/services/test_trigger_service.py

@@ -25,7 +25,7 @@ from mistral.services import security
 from mistral.services import triggers as t_s
 from mistral.services import workflows
 from mistral.tests.unit import base
-from mistral import utils
+from mistral_lib import utils
 
 # Use the set_default method to set value otherwise in certain test cases
 # the change in value is not permanent.

mistral/tests/unit/services/test_workflow_service.py

@@ -24,8 +24,8 @@ from mistral.lang.v2 import tasks
 from mistral.lang.v2 import workflows
 from mistral.services import workflows as wf_service
 from mistral.tests.unit import base
-from mistral import utils
 from mistral.workflow import states
+from mistral_lib import utils
 
 
 # Use the set_default method to set value otherwise in certain test cases

mistral/tests/unit/test_coordination.py

@@ -125,7 +125,8 @@ class ServiceTest(base.BaseTest):
         # new coordination configuration.
         coordination.cleanup_service_coordinator()
 
-    @mock.patch('mistral.utils.get_process_identifier', return_value='fake_id')
+    @mock.patch('mistral_lib.utils.get_process_identifier',
+                return_value='fake_id')
     def test_register_membership(self, mock_get_identifier):
         cfg.CONF.set_default('backend_url', 'zake://', 'coordination')
 

mistral/tests/unit/utils/test_utils.py

@@ -14,75 +14,13 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
-import copy
-
-import testtools.matchers as ttm
-
 from mistral import exceptions as exc
 from mistral.tests.unit import base
-from mistral import utils
 from mistral.utils import ssh_utils
-
-LEFT = {
-    'key1': {
-        'key11': "val11"
-    },
-    'key2': 'val2'
-}
-
-RIGHT = {
-    'key1': {
-        'key11': "val111111",
-        'key12': "val12",
-        'key13': {
-            'key131': 'val131'
-        }
-    },
-    'key2': 'val2222',
-    'key3': 'val3'
-}
+from mistral_lib import utils
 
 
 class UtilsTest(base.BaseTest):
-    def test_merge_dicts(self):
-        left = copy.deepcopy(LEFT)
-        right = copy.deepcopy(RIGHT)
-
-        expected = {
-            'key1': {
-                'key11': "val111111",
-                'key12': "val12",
-                'key13': {
-                    'key131': 'val131'
-                }
-            },
-            'key2': 'val2222',
-            'key3': 'val3'
-        }
-
-        utils.merge_dicts(left, right)
-
-        self.assertDictEqual(left, expected)
-
-    def test_merge_dicts_overwrite_false(self):
-        left = copy.deepcopy(LEFT)
-        right = copy.deepcopy(RIGHT)
-
-        expected = {
-            'key1': {
-                'key11': "val11",
-                'key12': "val12",
-                'key13': {
-                    'key131': 'val131'
-                }
-            },
-            'key2': 'val2',
-            'key3': 'val3'
-        }
-
-        utils.merge_dicts(left, right, overwrite=False)
-
-        self.assertDictEqual(left, expected)
 
     def test_itersubclasses(self):
         class A(object):
@@ -99,27 +37,6 @@ class UtilsTest(base.BaseTest):
 
         self.assertEqual([B, C, D], list(utils.iter_subclasses(A)))
 
-    def test_get_dict_from_entries(self):
-        input = ['param1', {'param2': 2}]
-        input_dict = utils.get_dict_from_entries(input)
-
-        self.assertIn('param1', input_dict)
-        self.assertIn('param2', input_dict)
-        self.assertEqual(2, input_dict.get('param2'))
-        self.assertIs(input_dict.get('param1'), utils.NotDefined)
-
-    def test_get_input_dict_from_string(self):
-        self.assertDictEqual(
-            {
-                'param1': utils.NotDefined,
-                'param2': 2,
-                'param3': 'var3'
-            },
-            utils.get_dict_from_string('param1, param2=2, param3="var3"')
-        )
-
-        self.assertDictEqual({}, utils.get_dict_from_string(''))
-
     def test_paramiko_to_private_key(self):
         self.assertRaises(
             exc.DataAccessException,
@@ -131,133 +48,3 @@ class UtilsTest(base.BaseTest):
             ssh_utils._to_paramiko_private_key,
             "..\\dir"
         )
-
-    def test_cut_string(self):
-        s = 'Hello, Mistral!'
-
-        self.assertEqual('Hello...', utils.cut_string(s, length=5))
-        self.assertEqual(s, utils.cut_string(s, length=100))
-        self.assertEqual(s, utils.cut_string(s, length=-1))
-
-    def test_cut_list(self):
-        l = ['Hello, Mistral!', 'Hello, OpenStack!']
-
-        self.assertEqual("['Hello, M...", utils.cut_list(l, 13))
-        self.assertEqual("['Hello, Mistr...", utils.cut_list(l, 17))
-        self.assertEqual("['Hello, Mistral!', 'He...", utils.cut_list(l, 26))
-
-        self.assertEqual(
-            "['Hello, Mistral!', 'Hello, OpenStack!']",
-            utils.cut_list(l, 100)
-        )
-
-        self.assertEqual(
-            "['Hello, Mistral!', 'Hello, OpenStack!']",
-            utils.cut_list(l, -1)
-        )
-
-        self.assertEqual("[1, 2...", utils.cut_list([1, 2, 3, 4, 5], 8))
-        self.assertEqual("[1, 2,...", utils.cut_list([1, 2, 3, 4, 5], 9))
-        self.assertEqual("[1, 2, 3...", utils.cut_list([1, 2, 3, 4, 5], 11))
-
-        self.assertRaises(ValueError, utils.cut_list, (1, 2))
-
-    def test_cut_list_with_large_dict_of_str(self):
-        d = [str(i) for i in range(65535)]
-        s = utils.cut(d, 65535)
-        self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
-
-    def test_cut_list_with_large_dict_of_int(self):
-        d = [i for i in range(65535)]
-        s = utils.cut(d, 65535)
-        self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
-
-    def test_cut_list_with_large_dict_of_dict(self):
-        d = [{'value': str(i)} for i in range(65535)]
-        s = utils.cut(d, 65535)
-        self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
-
-    def test_cut_list_for_state_info(self):
-        d = [{'value': 'This is a string that exceeds 35 characters'}
-             for i in range(2000)]
-        s = utils.cut(d, 65500)
-        self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65500)))
-
-    def test_cut_dict_with_strings(self):
-        d = {'key1': 'value1', 'key2': 'value2'}
-
-        s = utils.cut_dict(d, 13)
-
-        self.assertIn(s, ["{'key1': '...", "{'key2': '..."])
-
-        s = utils.cut_dict(d, 15)
-
-        self.assertIn(s, ["{'key1': 'va...", "{'key2': 'va..."])
-
-        s = utils.cut_dict(d, 22)
-
-        self.assertIn(
-            s,
-            ["{'key1': 'value1', ...", "{'key2': 'value2', ..."]
-        )
-
-        self.assertIn(
-            utils.cut_dict(d, 100),
-            [
-                "{'key1': 'value1', 'key2': 'value2'}",
-                "{'key2': 'value2', 'key1': 'value1'}"
-            ]
-        )
-
-        self.assertIn(
-            utils.cut_dict(d, -1),
-            [
-                "{'key1': 'value1', 'key2': 'value2'}",
-                "{'key2': 'value2', 'key1': 'value1'}"
-            ]
-        )
-
-        self.assertRaises(ValueError, utils.cut_dict, (1, 2))
-
-    def test_cut_dict_with_digits(self):
-        d = {1: 2, 3: 4}
-
-        s = utils.cut_dict(d, 10)
-
-        self.assertIn(s, ["{1: 2, ...", "{3: 4, ..."])
-
-        s = utils.cut_dict(d, 11)
-
-        self.assertIn(s, ["{1: 2, 3...", "{3: 4, 1..."])
-
-        s = utils.cut_dict(d, 100)
-
-        self.assertIn(s, ["{1: 2, 3: 4}", "{3: 4, 1: 2}"])
-
-    def test_cut_dict_with_large_dict_of_str(self):
-        d = {}
-        for i in range(65535):
-            d[str(i)] = str(i)
-        s = utils.cut(d, 65535)
-        self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
-
-    def test_cut_dict_with_large_dict_of_int(self):
-        d = {}
-        for i in range(65535):
-            d[i] = i
-        s = utils.cut(d, 65535)
-        self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
-
-    def test_cut_dict_with_large_dict_of_dict(self):
-        d = {}
-        for i in range(65535):
-            d[i] = {'value': str(i)}
-        s = utils.cut(d, 65535)
-        self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
-
-    def test_cut_dict_for_state_info(self):
-        d = {}
-        for i in range(2000):
-            d[i] = {'value': 'This is a string that exceeds 35 characters'}
-        s = utils.cut(d, 65500)
-        self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65500)))

mistral/utils/__init__.py

@@ -15,30 +15,15 @@
 #    limitations under the License.
 
 import contextlib
-import datetime
-import functools
-import json
 import os
-from os import path
 import shutil
-import socket
-import string
-import sys
 import tempfile
 import threading
 
-import eventlet
-from eventlet import corolocal
 from oslo_concurrency import processutils
-from oslo_log import log as logging
-from oslo_utils import timeutils
-from oslo_utils import uuidutils
-import pkg_resources as pkg
-import random
 
 from mistral import exceptions as exc
 
-
 # Thread local storage.
 _th_loc_storage = threading.local()
 
@@ -49,380 +34,6 @@ ACTION_TASK_TYPE = 'ACTION'
 WORKFLOW_TASK_TYPE = 'WORKFLOW'
 
 
-def generate_unicode_uuid():
-    return uuidutils.generate_uuid()
-
-
-def is_valid_uuid(uuid_string):
-    return uuidutils.is_uuid_like(uuid_string)
-
-
-def _get_greenlet_local_storage():
-    greenlet_id = corolocal.get_ident()
-
-    greenlet_locals = getattr(_th_loc_storage, "greenlet_locals", None)
-
-    if not greenlet_locals:
-        greenlet_locals = {}
-        _th_loc_storage.greenlet_locals = greenlet_locals
-
-    if greenlet_id in greenlet_locals:
-        return greenlet_locals[greenlet_id]
-    else:
-        return None
-
-
-def has_thread_local(var_name):
-    gl_storage = _get_greenlet_local_storage()
-    return gl_storage and var_name in gl_storage
-
-
-def get_thread_local(var_name):
-    if not has_thread_local(var_name):
-        return None
-
-    return _get_greenlet_local_storage()[var_name]
-
-
-def set_thread_local(var_name, val):
-    if val is None and has_thread_local(var_name):
-        gl_storage = _get_greenlet_local_storage()
-
-        # Delete variable from greenlet local storage.
-        if gl_storage:
-            del gl_storage[var_name]
-
-        # Delete the entire greenlet local storage from thread local storage.
-        if gl_storage and len(gl_storage) == 0:
-            del _th_loc_storage.greenlet_locals[corolocal.get_ident()]
-
-    if val is not None:
-        gl_storage = _get_greenlet_local_storage()
-        if not gl_storage:
-            gl_storage = _th_loc_storage.greenlet_locals[
-                corolocal.get_ident()] = {}
-
-        gl_storage[var_name] = val
-
-
-def log_exec(logger, level=logging.DEBUG):
-    """Decorator for logging function execution.
-
-        By default, target function execution is logged with DEBUG level.
-    """
-
-    def _decorator(func):
-        @functools.wraps(func)
-        def _logged(*args, **kw):
-            params_repr = ("[args=%s, kw=%s]" % (str(args), str(kw))
-                           if args or kw else "")
-
-            func_repr = ("Called method [name=%s, doc='%s', params=%s]" %
-                         (func.__name__, func.__doc__, params_repr))
-
-            logger.log(level, func_repr)
-
-            return func(*args, **kw)
-
-        _logged.__doc__ = func.__doc__
-
-        return _logged
-
-    return _decorator
-
-
-def merge_dicts(left, right, overwrite=True):
-    """Merges two dictionaries.
-
-    Values of right dictionary recursively get merged into left dictionary.
-    :param left: Left dictionary.
-    :param right: Right dictionary.
-    :param overwrite: If False, left value will not be overwritten if exists.
-    """
-
-    if left is None:
-        return right
-
-    if right is None:
-        return left
-
-    for k, v in right.items():
-        if k not in left:
-            left[k] = v
-        else:
-            left_v = left[k]
-
-            if isinstance(left_v, dict) and isinstance(v, dict):
-                merge_dicts(left_v, v, overwrite=overwrite)
-            elif overwrite:
-                left[k] = v
-
-    return left
-
-
-def update_dict(left, right):
-    """Updates left dict with content from right dict
-
-    :param left: Left dict.
-    :param right: Right dict.
-    :return: the updated left dictionary.
-    """
-
-    if left is None:
-        return right
-
-    if right is None:
-        return left
-
-    left.update(right)
-
-    return left
-
-
-def get_file_list(directory):
-    base_path = pkg.resource_filename("mistral", directory)
-
-    return [path.join(base_path, f) for f in os.listdir(base_path)
-            if path.isfile(path.join(base_path, f))]
-
-
-def cut_dict(d, length=100):
-    """Removes dictionary entries according to the given length.
-
-    This method removes a number of entries, if needed, so that a
-    string representation would fit into the given length.
-    The intention of this method is to optimize truncation of string
-    representation for dictionaries where the exact precision is not
-    critically important. Otherwise, we'd always have to convert a dict
-    into a string first and then shrink it to a needed size which will
-    increase memory footprint and reduce performance in case of large
-    dictionaries (i.e. tens of thousands entries).
-    Note that the method, due to complexity of the algorithm, has some
-    non-zero precision which depends on exact keys and values placed into
-    the dict. So for some dicts their reduced string representations will
-    be only approximately equal to the given value (up to around several
-    chars difference).
-
-    :param d: A dictionary.
-    :param length: A length limiting the dictionary string representation.
-    :return: A dictionary which is a subset of the given dictionary.
-    """
-
-    if not isinstance(d, dict):
-        raise ValueError("A dictionary is expected, got: %s" % type(d))
-
-    res = "{"
-
-    idx = 0
-
-    for key, value in d.items():
-        k = str(key)
-        v = str(value)
-
-        # Processing key.
-        new_len = len(k)
-
-        is_str = isinstance(key, str)
-
-        if is_str:
-            new_len += 2    # Account for the quotation marks
-
-        if 0 <= length <= new_len + len(res):
-            res += "'%s" % k if is_str else k
-            break
-        else:
-            res += "'%s'" % k if is_str else k
-
-        res += ": "
-
-        # Processing value.
-        new_len = len(v)
-
-        is_str = isinstance(value, str)
-
-        if is_str:
-            new_len += 2
-
-        if 0 <= length <= new_len + len(res):
-            res += "'%s" % v if is_str else v
-            break
-        else:
-            res += "'%s'" % v if is_str else v
-
-        res += ', ' if idx < len(d) - 1 else '}'
-
-        idx += 1
-
-    if 0 <= length <= len(res) and res[length - 1] is not '}':
-        res = res[:length - 3] + '...'
-
-    return res
-
-
-def cut_list(l, length=100):
-    if not isinstance(l, list):
-        raise ValueError("A list is expected, got: %s" % type(l))
-
-    res = '['
-
-    for idx, item in enumerate(l):
-        s = str(item)
-
-        new_len = len(res) + len(s)
-
-        is_str = isinstance(item, str)
-
-        if is_str:
-            new_len += 2
-
-        if 0 <= length <= new_len:
-            res += "'%s" % s if is_str else s
-            break
-        else:
-            res += "'%s'" % s if is_str else s
-        res += ', ' if idx < len(l) - 1 else ']'
-
-    if 0 <= length <= len(res) and res[length - 1] is not ']':
-        res = res[:length - 3] + '...'
-
-    return res
-
-
-def cut_string(s, length=100):
-    if 0 <= length < len(s):
-        return "%s..." % s[:length]
-
-    return s
-
-
-def cut(data, length=100):
-    if not data:
-        return data
-
-    if isinstance(data, list):
-        return cut_list(data, length=length)
-
-    if isinstance(data, dict):
-        return cut_dict(data, length=length)
-
-    return cut_string(str(data), length=length)
-
-
-def cut_by_kb(data, kilobytes):
-    length = get_number_of_chars_from_kilobytes(kilobytes)
-    return cut(data, length)
-
-
-def cut_by_char(data, length):
-    return cut(data, length)
-
-
-def iter_subclasses(cls, _seen=None):
-    """Generator over all subclasses of a given class in depth first order."""
-
-    if not isinstance(cls, type):
-        raise TypeError('iter_subclasses must be called with new-style class'
-                        ', not %.100r' % cls)
-    _seen = _seen or set()
-
-    try:
-        subs = cls.__subclasses__()
-    except TypeError:  # fails only when cls is type
-        subs = cls.__subclasses__(cls)
-
-    for sub in subs:
-        if sub not in _seen:
-            _seen.add(sub)
-            yield sub
-            for _sub in iter_subclasses(sub, _seen):
-                yield _sub
-
-
-def random_sleep(limit=1):
-    """Sleeps for a random period of time not exceeding the given limit.
-
-    Mostly intended to be used by tests to emulate race conditions.
-
-    :param limit: Float number of seconds that a sleep period must not exceed.
-    """
-
-    seconds = random.Random().randint(0, limit * 1000) * 0.001
-
-    print("Sleep: %s sec..." % seconds)
-
-    eventlet.sleep(seconds)
-
-
-class NotDefined(object):
-    """Marker of an empty value.
-
-    In a number of cases None can't be used to express the semantics of
-    a not defined value because None is just a normal value rather than
-    a value set to denote that it's not defined. This class can be used
-    in such cases instead of None.
-    """
-
-    pass
-
-
-def get_number_of_chars_from_kilobytes(kilobytes):
-    bytes_per_char = sys.getsizeof('s') - sys.getsizeof('')
-    total_number_of_chars = int(kilobytes * 1024 / bytes_per_char)
-    return total_number_of_chars
-
-
-def get_dict_from_string(string, delimiter=','):
-    if not string:
-        return {}
-
-    kv_dicts = []
-
-    for kv_pair_str in string.split(delimiter):
-        kv_str = kv_pair_str.strip()
-        kv_list = kv_str.split('=')
-
-        if len(kv_list) > 1:
-            try:
-                value = json.loads(kv_list[1])
-            except ValueError:
-                value = kv_list[1]
-
-            kv_dicts += [{kv_list[0]: value}]
-        else:
-            kv_dicts += [kv_list[0]]
-
-    return get_dict_from_entries(kv_dicts)
-
-
-def get_dict_from_entries(entries):
-    """Transforms a list of entries into dictionary.
-
-    :param entries: A list of entries.
-        If an entry is a dictionary the method simply updates the result
-        dictionary with its content.
-        If an entry is not a dict adds {entry, NotDefined} into the result.
-    """
-
-    result = {}
-
-    for e in entries:
-        if isinstance(e, dict):
-            result.update(e)
-        else:
-            # NOTE(kong): we put NotDefined here as the value of
-            # param without value specified, to distinguish from
-            # the valid values such as None, ''(empty string), etc.
-            result[e] = NotDefined
-
-    return result
-
-
-def get_process_identifier():
-    """Gets current running process identifier."""
-
-    return "%s_%s" % (socket.gethostname(), os.getpid())
-
-
 @contextlib.contextmanager
 def tempdir(**kwargs):
     argdict = kwargs.copy()
@@ -490,55 +101,3 @@ def generate_key_pair(key_length=2048):
         public_key = open(public_key_path).read()
 
         return private_key, public_key
-
-
-def utc_now_sec():
-    """Returns current time and drops microseconds."""
-
-    return drop_microseconds(timeutils.utcnow())
-
-
-def drop_microseconds(date):
-    """Drops microseconds and returns date."""
-    return date.replace(microsecond=0)
-
-
-def datetime_to_str(val, sep=' '):
-    """Converts datetime value to string.
-
-    If the given value is not an instance of datetime then the method
-    returns the same value.
-
-    :param val: datetime value.
-    :param sep: Separator between date and time.
-    :return: Datetime as a string.
-    """
-    if isinstance(val, datetime.datetime):
-        return val.isoformat(sep)
-
-    return val
-
-
-def datetime_to_str_in_dict(d, key, sep=' '):
-    """Converts datetime value in te given dict to string.
-
-    :param d: A dictionary.
-    :param key: The key for which we need to convert the value.
-    :param sep: Separator between date and time.
-    """
-    val = d.get(key)
-
-    if val is not None:
-        d[key] = datetime_to_str(d[key], sep=sep)
-
-
-def generate_string(length):
-    """Returns random string.
-
-    :param length: the length of returned string
-    """
-
-    return ''.join(random.choice(
-        string.ascii_uppercase + string.digits)
-        for _ in range(length)
-    )

mistral/utils/expression_utils.py

@@ -27,8 +27,8 @@ from yaql.language import utils as yaql_utils
 
 from mistral.config import cfg
 from mistral.db.v2 import api as db_api
-from mistral import utils
 from mistral.utils import filter_utils
+from mistral_lib import utils
 
 # TODO(rakhmerov): it's work around the bug in YAQL.
 # YAQL shouldn't expose internal types to custom functions.

mistral/utils/profiler.py

@@ -21,7 +21,7 @@ from oslo_log import log as logging
 import osprofiler.profiler
 import osprofiler.web
 
-from mistral import utils
+from mistral_lib import utils
 
 
 PROFILER_LOG = logging.getLogger(cfg.CONF.profiler.profiler_log_name)

mistral/workflow/base.py

@@ -23,10 +23,10 @@ from osprofiler import profiler
 from mistral.db.v2 import api as db_api
 from mistral import exceptions as exc
 from mistral.lang import parser as spec_parser
-from mistral import utils as u
 from mistral.workflow import commands
 from mistral.workflow import data_flow
 from mistral.workflow import states
+from mistral_lib import utils as u
 
 
 LOG = logging.getLogger(__name__)

mistral/workflow/data_flow.py

@@ -21,9 +21,9 @@ from mistral.db.v2.sqlalchemy import models
 from mistral import exceptions as exc
 from mistral import expressions as expr
 from mistral.lang import parser as spec_parser
-from mistral import utils
 from mistral.utils import inspect_utils
 from mistral.workflow import states
+from mistral_lib import utils
 
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF

mistral/workflow/direct_workflow.py

@@ -18,11 +18,11 @@ from osprofiler import profiler
 from mistral.db.v2 import api as db_api
 from mistral import exceptions as exc
 from mistral import expressions as expr
-from mistral import utils
 from mistral.workflow import base
 from mistral.workflow import commands
 from mistral.workflow import data_flow
 from mistral.workflow import states
+from mistral_lib import utils
 
 
 LOG = logging.getLogger(__name__)

requirements.txt

@@ -13,7 +13,7 @@ gnocchiclient>=3.3.1 # Apache-2.0
 Jinja2>=2.10 # BSD License (3 clause)
 #jsonschema>=2.6.0 # MIT
 keystonemiddleware>=4.18.0 # Apache-2.0
-mistral-lib>=0.4.0 # Apache-2.0
+mistral-lib>=1.2.0 # Apache-2.0
 networkx<2.3,>=1.10;python_version<'3.0'  # BSD
 networkx>=2.3;python_version>='3.4'  # BSD
 oslo.concurrency>=3.26.0 # Apache-2.0

test-requirements.txt

@@ -8,7 +8,7 @@ croniter>=0.3.4 # MIT License
 doc8>=0.6.0 # Apache-2.0
 fixtures>=3.0.0 # Apache-2.0/BSD
 keystonemiddleware>=4.18.0 # Apache-2.0
-mistral-lib>=0.4.0 # Apache-2.0
+mistral-lib>=1.2.0 # Apache-2.0
 mock>=2.0.0 # BSD
 networkx<2.3,>=1.10;python_version<'3.0'  # BSD
 networkx>=2.3;python_version>='3.4'  # BSD