From e5cec09f58f1145b88ee587ac0e6de93ddab50bb Mon Sep 17 00:00:00 2001
From: Kaustuv Royburman <kaustuv.royburman@ericsson.com>
Date: Sat, 2 Sep 2017 23:15:50 -0500
Subject: [PATCH] Decoupling of Mistral tempest test from Mistral code base

The Mistral Tempest tests have a hard-coded dependency on Mistral being present
when Tempest tests are executed.
When trying to sparse-checkout the mistral_tempest_tests folder to run
the Mistral tests as a Tempest plugin; it fails due to Mistral not
being installed as some utilities and resources which are written in
the Mistral Tempest tests are being hard referenced from Mistral
being installed in the same environment.

This patch decouples the Mistral Tempest Tests so that they can be executed in
a stand-alone mode along with the necessary resources that are required to
execute the Tempest tests.

Change-Id: Ifd6a3a65a14c4ad4736dccc3e72cd564b6f53a0a
Closes-Bug: #1714732
---
 mistral_tempest_tests/services/base.py        |   4 +-
 .../tests/api/v2/test_actions.py              |   2 +-
 .../tests/api/v2/test_executions.py           |   2 +-
 .../tests/api/v2/test_workflows.py            |   2 +-
 .../tests/resources/action_v2.yaml            |  21 +++
 .../for_wf_namespace/lowest_level_wf.yaml     |   6 +
 .../resources/for_wf_namespace/middle_wf.yaml |   6 +
 .../for_wf_namespace/top_level_wf.yaml        |   6 +
 .../openstack/action_collection_wb.yaml       |  53 ++++++
 .../tests/resources/single_wf.yaml            |  11 ++
 .../tests/resources/wb_v1.yaml                |  12 ++
 .../tests/resources/wb_v2.yaml                |  13 ++
 .../tests/resources/wb_with_nested_wf.yaml    |  18 +++
 .../resources/wf_action_ex_concurrency.yaml   |   8 +
 .../resources/wf_task_ex_concurrency.yaml     |  11 ++
 .../tests/resources/wf_v2.yaml                |  34 ++++
 .../engine/actions/v2/test_ssh_actions.py     |   4 +-
 mistral_tempest_tests/tests/ssh_utils.py      | 103 ++++++++++++
 mistral_tempest_tests/tests/utils.py          | 152 ++++++++++++++++++
 19 files changed, 462 insertions(+), 6 deletions(-)
 create mode 100755 mistral_tempest_tests/tests/resources/action_v2.yaml
 create mode 100755 mistral_tempest_tests/tests/resources/for_wf_namespace/lowest_level_wf.yaml
 create mode 100755 mistral_tempest_tests/tests/resources/for_wf_namespace/middle_wf.yaml
 create mode 100755 mistral_tempest_tests/tests/resources/for_wf_namespace/top_level_wf.yaml
 create mode 100755 mistral_tempest_tests/tests/resources/openstack/action_collection_wb.yaml
 create mode 100755 mistral_tempest_tests/tests/resources/single_wf.yaml
 create mode 100755 mistral_tempest_tests/tests/resources/wb_v1.yaml
 create mode 100755 mistral_tempest_tests/tests/resources/wb_v2.yaml
 create mode 100755 mistral_tempest_tests/tests/resources/wb_with_nested_wf.yaml
 create mode 100755 mistral_tempest_tests/tests/resources/wf_action_ex_concurrency.yaml
 create mode 100755 mistral_tempest_tests/tests/resources/wf_task_ex_concurrency.yaml
 create mode 100755 mistral_tempest_tests/tests/resources/wf_v2.yaml
 create mode 100755 mistral_tempest_tests/tests/ssh_utils.py
 create mode 100755 mistral_tempest_tests/tests/utils.py

diff --git a/mistral_tempest_tests/services/base.py b/mistral_tempest_tests/services/base.py
index 8f42b65..5ee7d51 100644
--- a/mistral_tempest_tests/services/base.py
+++ b/mistral_tempest_tests/services/base.py
@@ -30,7 +30,9 @@ def get_resource(path):
     main_package = 'mistral_tempest_tests'
     dir_path = __file__[0:__file__.find(main_package)]
 
-    return open(dir_path + 'mistral/tests/resources/' + path).read()
+    return open(dir_path +
+                'mistral_tempest_tests/tests/resources/' +
+                path).read()
 
 
 def find_items(items, **props):
diff --git a/mistral_tempest_tests/tests/api/v2/test_actions.py b/mistral_tempest_tests/tests/api/v2/test_actions.py
index 8e0e2a0..0ee35df 100644
--- a/mistral_tempest_tests/tests/api/v2/test_actions.py
+++ b/mistral_tempest_tests/tests/api/v2/test_actions.py
@@ -16,8 +16,8 @@ import datetime
 from tempest.lib import decorators
 from tempest.lib import exceptions
 
-from mistral import utils
 from mistral_tempest_tests.tests import base
+from mistral_tempest_tests.tests import utils
 
 
 class ActionTestsV2(base.TestCase):
diff --git a/mistral_tempest_tests/tests/api/v2/test_executions.py b/mistral_tempest_tests/tests/api/v2/test_executions.py
index e797531..96abe43 100644
--- a/mistral_tempest_tests/tests/api/v2/test_executions.py
+++ b/mistral_tempest_tests/tests/api/v2/test_executions.py
@@ -16,8 +16,8 @@ from oslo_concurrency.fixture import lockutils
 from tempest.lib import decorators
 from tempest.lib import exceptions
 
-from mistral import utils
 from mistral_tempest_tests.tests import base
+from mistral_tempest_tests.tests import utils
 
 import json
 
diff --git a/mistral_tempest_tests/tests/api/v2/test_workflows.py b/mistral_tempest_tests/tests/api/v2/test_workflows.py
index f9bc6cd..2b3c1c1 100644
--- a/mistral_tempest_tests/tests/api/v2/test_workflows.py
+++ b/mistral_tempest_tests/tests/api/v2/test_workflows.py
@@ -17,8 +17,8 @@ from oslo_concurrency.fixture import lockutils
 from tempest.lib import decorators
 from tempest.lib import exceptions
 
-from mistral import utils
 from mistral_tempest_tests.tests import base
+from mistral_tempest_tests.tests import utils
 
 
 class WorkflowTestsV2(base.TestCase):
diff --git a/mistral_tempest_tests/tests/resources/action_v2.yaml b/mistral_tempest_tests/tests/resources/action_v2.yaml
new file mode 100755
index 0000000..bf2b879
--- /dev/null
+++ b/mistral_tempest_tests/tests/resources/action_v2.yaml
@@ -0,0 +1,21 @@
+---
+version: "2.0"
+
+greeting:
+  description: "This action says 'Hello'"
+  tags: [hello]
+  base: std.echo
+  base-input:
+    output: 'Hello,  <% $.name %>'
+  input:
+    - name
+  output:
+    string: <% $ %>
+
+farewell:
+  base: std.echo
+  base-input:
+    output: 'Bye!'
+  output:
+    info: <% $ %>
+
diff --git a/mistral_tempest_tests/tests/resources/for_wf_namespace/lowest_level_wf.yaml b/mistral_tempest_tests/tests/resources/for_wf_namespace/lowest_level_wf.yaml
new file mode 100755
index 0000000..5d873f2
--- /dev/null
+++ b/mistral_tempest_tests/tests/resources/for_wf_namespace/lowest_level_wf.yaml
@@ -0,0 +1,6 @@
+---
+version: '2.0'
+lowest_level_wf:
+  tasks:
+    noop_task:
+      action: std.noop
\ No newline at end of file
diff --git a/mistral_tempest_tests/tests/resources/for_wf_namespace/middle_wf.yaml b/mistral_tempest_tests/tests/resources/for_wf_namespace/middle_wf.yaml
new file mode 100755
index 0000000..e0cc295
--- /dev/null
+++ b/mistral_tempest_tests/tests/resources/for_wf_namespace/middle_wf.yaml
@@ -0,0 +1,6 @@
+---
+version: '2.0'
+middle_wf:
+  tasks:
+    run_workflow_with_name_lowest_level_wf:
+      workflow: lowest_level_wf
\ No newline at end of file
diff --git a/mistral_tempest_tests/tests/resources/for_wf_namespace/top_level_wf.yaml b/mistral_tempest_tests/tests/resources/for_wf_namespace/top_level_wf.yaml
new file mode 100755
index 0000000..2bedcb1
--- /dev/null
+++ b/mistral_tempest_tests/tests/resources/for_wf_namespace/top_level_wf.yaml
@@ -0,0 +1,6 @@
+---
+version: '2.0'
+top_level_wf:
+  tasks:
+    run_workflow_with_name_middle_wf:
+      workflow: middle_wf
\ No newline at end of file
diff --git a/mistral_tempest_tests/tests/resources/openstack/action_collection_wb.yaml b/mistral_tempest_tests/tests/resources/openstack/action_collection_wb.yaml
new file mode 100755
index 0000000..d675a58
--- /dev/null
+++ b/mistral_tempest_tests/tests/resources/openstack/action_collection_wb.yaml
@@ -0,0 +1,53 @@
+---
+version: '2.0'
+name: action_collection
+
+workflows:
+  keystone:
+    type: direct
+    tasks:
+      projects_list:
+        action: keystone.projects_list
+        publish:
+           result: <% task().result %>
+
+  nova:
+    type: direct
+    tasks:
+      flavors_list:
+        action: nova.flavors_list
+        publish:
+           result: <% task().result %>
+
+  glance:
+    type: direct
+    tasks:
+      images_list:
+        action: glance.images_list
+        publish:
+           result: <% task().result %>
+
+  heat:
+    type: direct
+    tasks:
+      stacks_list:
+        action: heat.stacks_list
+        publish:
+           result: <% task().result %>
+
+  neutron:
+    type: direct
+    tasks:
+      list_subnets:
+        action: neutron.list_subnets
+        publish:
+           result: <% task().result %>
+
+  cinder:
+    type: direct
+    tasks:
+      volumes_list:
+        action: cinder.volumes_list
+        publish:
+           result: <% task().result %>
+
diff --git a/mistral_tempest_tests/tests/resources/single_wf.yaml b/mistral_tempest_tests/tests/resources/single_wf.yaml
new file mode 100755
index 0000000..1dc2c6b
--- /dev/null
+++ b/mistral_tempest_tests/tests/resources/single_wf.yaml
@@ -0,0 +1,11 @@
+---
+version: '2.0'
+
+single_wf:
+  type: direct
+
+  tasks:
+    hello:
+      action: std.echo output="Hello"
+      publish:
+        result: <% task(hello).result %>
diff --git a/mistral_tempest_tests/tests/resources/wb_v1.yaml b/mistral_tempest_tests/tests/resources/wb_v1.yaml
new file mode 100755
index 0000000..dc47029
--- /dev/null
+++ b/mistral_tempest_tests/tests/resources/wb_v1.yaml
@@ -0,0 +1,12 @@
+Namespaces:
+  Greetings:
+    actions:
+      hello:
+        class: std.echo
+        base-parameters:
+          output: Hello!
+
+Workflow:
+  tasks:
+    hello:
+      action: Greetings.hello
\ No newline at end of file
diff --git a/mistral_tempest_tests/tests/resources/wb_v2.yaml b/mistral_tempest_tests/tests/resources/wb_v2.yaml
new file mode 100755
index 0000000..1c0cd3b
--- /dev/null
+++ b/mistral_tempest_tests/tests/resources/wb_v2.yaml
@@ -0,0 +1,13 @@
+---
+version: '2.0'
+name: test
+
+workflows:
+  test:
+    type: direct
+
+    tasks:
+      hello:
+        action: std.echo output="Hello"
+        publish:
+           result: <% task(hello).result %>
diff --git a/mistral_tempest_tests/tests/resources/wb_with_nested_wf.yaml b/mistral_tempest_tests/tests/resources/wb_with_nested_wf.yaml
new file mode 100755
index 0000000..717855a
--- /dev/null
+++ b/mistral_tempest_tests/tests/resources/wb_with_nested_wf.yaml
@@ -0,0 +1,18 @@
+---
+version: "2.0"
+
+name: wb_with_nested_wf
+
+workflows:
+
+  wrapping_wf:
+    type: direct
+    tasks:
+      call_inner_wf:
+        workflow: inner_wf
+
+  inner_wf:
+    type: direct
+    tasks:
+      hello:
+        action: std.echo output="Hello from inner workflow"
\ No newline at end of file
diff --git a/mistral_tempest_tests/tests/resources/wf_action_ex_concurrency.yaml b/mistral_tempest_tests/tests/resources/wf_action_ex_concurrency.yaml
new file mode 100755
index 0000000..0dee550
--- /dev/null
+++ b/mistral_tempest_tests/tests/resources/wf_action_ex_concurrency.yaml
@@ -0,0 +1,8 @@
+---
+version: '2.0'
+
+test_action_ex_concurrency:
+  tasks:
+    test_with_items:
+      with-items: index in <% range(2) %>
+      action: std.echo output='<% $.index %>'
\ No newline at end of file
diff --git a/mistral_tempest_tests/tests/resources/wf_task_ex_concurrency.yaml b/mistral_tempest_tests/tests/resources/wf_task_ex_concurrency.yaml
new file mode 100755
index 0000000..4589408
--- /dev/null
+++ b/mistral_tempest_tests/tests/resources/wf_task_ex_concurrency.yaml
@@ -0,0 +1,11 @@
+---
+version: '2.0'
+
+test_task_ex_concurrency:
+  tasks:
+    task1:
+      action: std.async_noop
+      timeout: 2
+    task2:
+      action: std.async_noop
+      timeout: 2
\ No newline at end of file
diff --git a/mistral_tempest_tests/tests/resources/wf_v2.yaml b/mistral_tempest_tests/tests/resources/wf_v2.yaml
new file mode 100755
index 0000000..37bedbb
--- /dev/null
+++ b/mistral_tempest_tests/tests/resources/wf_v2.yaml
@@ -0,0 +1,34 @@
+---
+version: '2.0'
+
+wf:
+  type: direct
+
+  tasks:
+    hello:
+      action: std.echo output="Hello"
+      wait-before: 1
+      publish:
+        result: <% task(hello).result %>
+
+wf1:
+  type: reverse
+  input:
+    - farewell
+
+  tasks:
+    addressee:
+      action: std.echo output="John"
+      publish:
+        name: <% task(addressee).result %>
+
+    goodbye:
+      action: std.echo output="<% $.farewell %>, <% $.name %>"
+      requires: [addressee]
+
+wf2:
+  type: direct
+
+  tasks:
+    hello:
+      action: std.echo output="Doe"
diff --git a/mistral_tempest_tests/tests/scenario/engine/actions/v2/test_ssh_actions.py b/mistral_tempest_tests/tests/scenario/engine/actions/v2/test_ssh_actions.py
index 38e3231..32534ed 100644
--- a/mistral_tempest_tests/tests/scenario/engine/actions/v2/test_ssh_actions.py
+++ b/mistral_tempest_tests/tests/scenario/engine/actions/v2/test_ssh_actions.py
@@ -23,9 +23,9 @@ from tempest import config
 from tempest.lib import decorators
 from tempest.lib import exceptions
 
-from mistral import utils
-from mistral.utils import ssh_utils
 from mistral_tempest_tests.tests import base
+from mistral_tempest_tests.tests import ssh_utils
+from mistral_tempest_tests.tests import utils
 
 
 LOG = logging.getLogger(__name__)
diff --git a/mistral_tempest_tests/tests/ssh_utils.py b/mistral_tempest_tests/tests/ssh_utils.py
new file mode 100755
index 0000000..a8f9feb
--- /dev/null
+++ b/mistral_tempest_tests/tests/ssh_utils.py
@@ -0,0 +1,103 @@
+# Copyright 2014 - Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+from os import path
+from oslo_log import log as logging
+import paramiko
+import six
+
+KEY_PATH = path.expanduser("~/.ssh/")
+LOG = logging.getLogger(__name__)
+
+
+def _read_paramimko_stream(recv_func):
+    result = ''
+    buf = recv_func(1024)
+    while buf != '':
+        result += buf
+        buf = recv_func(1024)
+
+    return result
+
+
+def _to_paramiko_private_key(private_key_filename, password=None):
+    if '../' in private_key_filename or '..\\' in private_key_filename:
+        raise OSError(
+            "Private key filename must not contain '..'. "
+            "Actual: %s" % private_key_filename
+        )
+
+    private_key_path = KEY_PATH + private_key_filename
+
+    return paramiko.RSAKey(
+        filename=private_key_path,
+        password=password
+    )
+
+
+def _connect(host, username, password=None, pkey=None, proxy=None):
+    if isinstance(pkey, six.string_types):
+        pkey = _to_paramiko_private_key(pkey, password)
+
+    LOG.debug('Creating SSH connection to %s', host)
+
+    ssh_client = paramiko.SSHClient()
+    ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+    ssh_client.connect(
+        host,
+        username=username,
+        password=password,
+        pkey=pkey,
+        sock=proxy
+    )
+
+    return ssh_client
+
+
+def _cleanup(ssh_client):
+    ssh_client.close()
+
+
+def _execute_command(ssh_client, cmd, get_stderr=False,
+                     raise_when_error=True):
+    try:
+        chan = ssh_client.get_transport().open_session()
+        chan.exec_command(cmd)
+
+        # TODO(nmakhotkin): that could hang if stderr buffer overflows
+        stdout = _read_paramimko_stream(chan.recv)
+        stderr = _read_paramimko_stream(chan.recv_stderr)
+
+        ret_code = chan.recv_exit_status()
+
+        if ret_code and raise_when_error:
+            raise RuntimeError("Cmd: %s\nReturn code: %s\nstdout: %s"
+                               % (cmd, ret_code, stdout))
+        if get_stderr:
+            return ret_code, stdout, stderr
+        else:
+            return ret_code, stdout
+    finally:
+        _cleanup(ssh_client)
+
+
+def execute_command(cmd, host, username, password=None,
+                    private_key_filename=None, get_stderr=False,
+                    raise_when_error=True):
+    ssh_client = _connect(host, username, password, private_key_filename)
+
+    LOG.debug("Executing command %s", cmd)
+
+    return _execute_command(ssh_client, cmd, get_stderr, raise_when_error)
diff --git a/mistral_tempest_tests/tests/utils.py b/mistral_tempest_tests/tests/utils.py
new file mode 100755
index 0000000..05487b7
--- /dev/null
+++ b/mistral_tempest_tests/tests/utils.py
@@ -0,0 +1,152 @@
+# Copyright 2013 - Mirantis, Inc.
+# Copyright 2015 - Huawei Technologies Co. Ltd
+# Copyright 2016 - Brocade Communications Systems, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import contextlib
+import json
+import os
+import shutil
+import tempfile
+
+from oslo_concurrency import processutils
+
+
+class NotDefined(object):
+    """Marker of an empty value.
+
+    In a number of cases None can't be used to express the semantics of
+    a not defined value because None is just a normal value rather than
+    a value set to denote that it's not defined. This class can be used
+    in such cases instead of None.
+    """
+
+    pass
+
+
+def get_dict_from_string(string, delimiter=','):
+    if not string:
+        return {}
+
+    kv_dicts = []
+
+    for kv_pair_str in string.split(delimiter):
+        kv_str = kv_pair_str.strip()
+        kv_list = kv_str.split('=')
+
+        if len(kv_list) > 1:
+            try:
+                value = json.loads(kv_list[1])
+            except ValueError:
+                value = kv_list[1]
+
+            kv_dicts += [{kv_list[0]: value}]
+        else:
+            kv_dicts += [kv_list[0]]
+
+    return get_dict_from_entries(kv_dicts)
+
+
+def get_dict_from_entries(entries):
+    """Transforms a list of entries into dictionary.
+
+    :param entries: A list of entries.
+        If an entry is a dictionary the method simply updates the result
+        dictionary with its content.
+        If an entry is not a dict adds {entry, NotDefined} into the result.
+    """
+
+    result = {}
+
+    for e in entries:
+        if isinstance(e, dict):
+            result.update(e)
+        else:
+            # NOTE(kong): we put NotDefined here as the value of
+            # param without value specified, to distinguish from
+            # the valid values such as None, ''(empty string), etc.
+            result[e] = NotDefined
+
+    return result
+
+
+@contextlib.contextmanager
+def tempdir(**kwargs):
+    argdict = kwargs.copy()
+
+    if 'dir' not in argdict:
+        argdict['dir'] = '/tmp/'
+
+    tmpdir = tempfile.mkdtemp(**argdict)
+
+    try:
+        yield tmpdir
+    finally:
+        try:
+            shutil.rmtree(tmpdir)
+        except OSError as e:
+            raise OSError(
+                "Failed to delete temp dir %(dir)s (reason: %(reason)s)" %
+                {'dir': tmpdir, 'reason': e}
+            )
+
+
+def save_text_to(text, file_path, overwrite=False):
+    if os.path.exists(file_path) and not overwrite:
+        raise OSError(
+            "Cannot save data to file. File %s already exists."
+        )
+
+    with open(file_path, 'w') as f:
+        f.write(text)
+
+
+def generate_key_pair(key_length=2048):
+    """Create RSA key pair with specified number of bits in key.
+
+    Returns tuple of private and public keys.
+    """
+    with tempdir() as tmpdir:
+        keyfile = os.path.join(tmpdir, 'tempkey')
+        args = [
+            'ssh-keygen',
+            '-q',  # quiet
+            '-N', '',  # w/o passphrase
+            '-t', 'rsa',  # create key of rsa type
+            '-f', keyfile,  # filename of the key file
+            '-C', 'Generated-by-Mistral'  # key comment
+        ]
+
+        if key_length is not None:
+            args.extend(['-b', key_length])
+
+        processutils.execute(*args)
+
+        if not os.path.exists(keyfile):
+            # raise exc.DataAccessException(
+            #     "Private key file hasn't been created"
+            # )
+            raise OSError("Private key file hasn't been created")
+
+        private_key = open(keyfile).read()
+        public_key_path = keyfile + '.pub'
+
+        if not os.path.exists(public_key_path):
+            # raise exc.DataAccessException(
+            #     "Public key file hasn't been created"
+            # )
+            raise OSError("Private key file hasn't been created")
+        public_key = open(public_key_path).read()
+
+        return private_key, public_key