Use mature forks instead of old originals

Forks is better tested and have wider usage.
[x] logwrap (`logwrap`, `pretty_repr`)
[x] exec-helpers (`Subprocess`, `SSHClient`, `SSHAuth`)
[x] threaded (`threaded`)

Move requirements to requirements.txt for reducing tests pain

Change-Id: I9175cd69599987a55d8be4cd9cb4b9464b4154ec
Signed-off-by: Alexey Stepanov <penguinolog@gmail.com>

devops/client/environment.py

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import exec_helpers
 import paramiko
 # pylint: disable=redefined-builtin
 # noinspection PyUnresolvedReferences
@@ -22,7 +23,6 @@ from devops.client import nailgun
 from devops import error
 from devops.helpers import helpers
 from devops.helpers import ntp
-from devops.helpers import ssh_client
 from devops.helpers import templates
 from devops import settings
 
@@ -135,9 +135,9 @@ class DevopsEnvironment(object):
             host=admin_ip, port=admin_node.ssh_port, timeout=180,
             timeout_msg=("Admin node {ip} is not accessible by SSH."
                          "".format(ip=admin_ip)))
-        return ssh_client.SSHClient(
+        return exec_helpers.SSHClient(
             admin_ip,
-            auth=ssh_client.SSHAuth(username=login, password=password))
+            auth=exec_helpers.SSHAuth(username=login, password=password))
 
     def get_private_keys(self):
         ssh_keys = []
@@ -165,9 +165,9 @@ class DevopsEnvironment(object):
         helpers.wait_tcp(
             host=ip, port=node.ssh_port, timeout=180,
             timeout_msg="Node {ip} is not accessible by SSH.".format(ip=ip))
-        return ssh_client.SSHClient(
+        return exec_helpers.SSHClient(
             ip,
-            auth=ssh_client.SSHAuth(
+            auth=exec_helpers.SSHAuth(
                 username=login,
                 password=password,
                 keys=self.get_private_keys()))
@@ -191,9 +191,9 @@ class DevopsEnvironment(object):
                          login=settings.SSH_SLAVE_CREDENTIALS['login'],
                          password=settings.SSH_SLAVE_CREDENTIALS['password']):
         ip = self.find_node_ip(node_name)
-        return ssh_client.SSHClient(
+        return exec_helpers.SSHClient(
             ip,
-            auth=ssh_client.SSHAuth(
+            auth=exec_helpers.SSHAuth(
                 username=login,
                 password=password))
 

devops/driver/libvirt/libvirt_driver.py

@@ -25,6 +25,7 @@ import xml.etree.ElementTree as ET
 
 from django.conf import settings
 from django.utils import functional
+import exec_helpers
 import libvirt
 import netaddr
 import paramiko
@@ -37,8 +38,6 @@ from devops.helpers import cloud_image_settings
 from devops.helpers import decorators
 from devops.helpers import helpers
 from devops.helpers import scancodes
-from devops.helpers import ssh_client
-from devops.helpers import subprocess_runner
 from devops import logger
 from devops.models import base
 from devops.models import driver
@@ -355,16 +354,16 @@ class LibvirtDriver(driver.Driver):
                 logger.debug("Initializing SSHClient for username:'{0}', host:"
                              "'{1}', port:'{2}'".format(username, host, port))
                 keys = [key]
-            return ssh_client.SSHClient(
+            return exec_helpers.SSHClient(
                 host=host,
                 port=port,
-                auth=ssh_client.SSHAuth(
+                auth=exec_helpers.SSHAuth(
                     username=username,
                     keys=keys))
         else:
             # Using SubprocessClient to execute shell commands on local host
             logger.debug("Initializing subprocess_runner for local host")
-            return subprocess_runner.Subprocess
+            return exec_helpers.Subprocess()
 
 
 class LibvirtL2NetworkDevice(network.L2NetworkDevice):
@@ -696,7 +695,7 @@ class LibvirtL2NetworkDevice(network.L2NetworkDevice):
             # of this try/except workaround
             try:
                 self.driver.shell.check_call(cmd)
-            except error.DevopsCalledProcessError:
+            except exec_helpers.CalledProcessError:
                 pass
 
     @decorators.retry(libvirt.libvirtError)

devops/driver/libvirt/libvirt_xml_builder.py

@@ -15,10 +15,14 @@ from __future__ import unicode_literals
 
 import hashlib
 
+import logwrap
 import six
 
-from devops.helpers.decorators import logwrap
 from devops.helpers import xmlgenerator
+from devops import logger
+
+
+log_call = logwrap.logwrap(log=logger)
 
 
 class LibvirtXMLBuilder(object):
@@ -36,7 +40,7 @@ class LibvirtXMLBuilder(object):
         return name
 
     @classmethod
-    @logwrap
+    @log_call
     def build_network_xml(cls, network_name, bridge_name, addresses=None,
                           forward=None, ip_network_address=None,
                           ip_network_prefixlen=None, stp=True,
@@ -92,7 +96,7 @@ class LibvirtXMLBuilder(object):
         return str(network_xml)
 
     @classmethod
-    @logwrap
+    @log_call
     def build_volume_xml(cls, name, capacity, vol_format, backing_store_path,
                          backing_store_format):
         """Generate volume XML
@@ -112,7 +116,7 @@ class LibvirtXMLBuilder(object):
         return str(volume_xml)
 
     @classmethod
-    @logwrap
+    @log_call
     def build_snapshot_xml(cls, name=None, description=None,
                            external=False, disk_only=False, memory_file='',
                            domain_isactive=False, local_disk_devices=None):
@@ -198,7 +202,7 @@ class LibvirtXMLBuilder(object):
                 device_xml.filterref(filter=interface_filter)
 
     @classmethod
-    @logwrap
+    @log_call
     def build_network_filter(cls, name, uuid=None, rule=None):
         """Generate nwfilter XML for network
 
@@ -216,7 +220,7 @@ class LibvirtXMLBuilder(object):
         return str(filter_xml)
 
     @classmethod
-    @logwrap
+    @log_call
     def build_interface_filter(cls, name, filterref, uuid=None, rule=None):
         """Generate nwfilter XML for interface
 
@@ -236,7 +240,7 @@ class LibvirtXMLBuilder(object):
         return str(filter_xml)
 
     @classmethod
-    @logwrap
+    @log_call
     def build_node_xml(cls, name, hypervisor, use_host_cpu, vcpu, memory,
                        use_hugepages, hpet, os_type, architecture, boot,
                        reboot_timeout, bootmenu_timeout, emulator,
@@ -333,7 +337,7 @@ class LibvirtXMLBuilder(object):
         return str(node_xml)
 
     @classmethod
-    @logwrap
+    @log_call
     def build_iface_xml(cls, name, ip=None, prefix=None, vlanid=None):
         """Generate interface bridge XML
 

devops/error.py

@@ -17,7 +17,7 @@ from __future__ import unicode_literals
 import inspect
 import warnings
 
-import six
+import exec_helpers
 
 
 class DevopsException(Exception):
@@ -35,36 +35,7 @@ class AuthenticationError(DevopsError):
     pass
 
 
-class DevopsCalledProcessError(DevopsError):
+class DevopsCalledProcessError(DevopsError, exec_helpers.CalledProcessError):
-    @staticmethod
-    def _makestr(data):
-        if isinstance(data, six.binary_type):
-            return data.decode('utf-8', errors='backslashreplace')
-        elif isinstance(data, six.text_type):
-            return data
-        else:
-            return repr(data)
-
-    def __init__(
-            self, command, returncode, expected=0, stdout=None, stderr=None):
-        self.returncode = returncode
-        self.expected = expected
-        self.cmd = command
-        self.stdout = stdout
-        self.stderr = stderr
-        message = (
-            "Command '{cmd}' returned exit code {code} while "
-            "expected {expected}".format(
-                cmd=self._makestr(self.cmd),
-                code=self.returncode,
-                expected=self.expected
-            ))
-        if self.stdout:
-            message += "\n\tSTDOUT:\n{}".format(self._makestr(self.stdout))
-        if self.stderr:
-            message += "\n\tSTDERR:\n{}".format(self._makestr(self.stderr))
-        super(DevopsCalledProcessError, self).__init__(message)
-
     @property
     def output(self):
         warnings.warn(

devops/helpers/cloud_image_settings.py

@@ -14,9 +14,9 @@
 
 import os
 
-from devops.helpers.helpers import format_data
+import exec_helpers
-from devops.helpers import subprocess_runner
 
+from devops.helpers.helpers import format_data
 from devops import logger
 
 
@@ -92,4 +92,4 @@ def generate_cloud_image_settings(cloud_image_settings_path, meta_data_path,
                                user_data_path,
                                meta_data_path)
 
-    subprocess_runner.Subprocess.check_call(cmd)
+    exec_helpers.Subprocess().check_call(cmd)

devops/helpers/decorators.py

@@ -19,11 +19,13 @@ import functools
 import inspect
 import logging
 import sys
-import threading
 import time
+import warnings
 
 import fasteners
+import logwrap as ext_logwrap
 import six
+import threaded as ext_threaded
 
 from devops import error
 from devops import logger
@@ -31,43 +33,12 @@ from devops import settings
 
 
 def threaded(name=None, started=False, daemon=False):
-    """Make function or method threaded with passing arguments
+    warnings.warn(
-
+        'helpers.decorators.threaded is deprecated'
-    If decorator added not as function, name is generated from function name.
+        ' in favor of external threaded',
-
+        DeprecationWarning
-    :type name: str
+    )
-    :type started: bool
+    return ext_threaded.threaded(name=name, started=started, daemon=daemon)
-    :type daemon: bool
-    """
-
-    def real_decorator(func):
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            """Thread generator for function
-
-            :rtype: Thread
-            """
-            if name is None:
-                func_name = 'Threaded {}'.format(func.__name__)
-            else:
-                func_name = name
-            thread = threading.Thread(
-                target=func,
-                name=func_name,
-                args=args,
-                kwargs=kwargs)
-            if daemon:
-                thread.daemon = True
-            if started:
-                thread.start()
-            return thread
-        return wrapper
-
-    if name is not None and callable(name):
-        func, name = name, None
-        return real_decorator(func)
-
-    return real_decorator
 
 
 def retry(exception, count=10, delay=1):
@@ -177,148 +148,32 @@ def _getcallargs(func, *positional, **named):
 # pylint:enable=no-member
 
 
-def _simple(item):
-    """Check for nested iterations: True, if not"""
-    return not isinstance(item, (list, set, tuple, dict))
-
-
-_formatters = {
-    'simple': "{spc:<{indent}}{val!r}".format,
-    'text': "{spc:<{indent}}{prefix}'''{string}'''".format,
-    'dict': "\n{spc:<{indent}}{key!r:{size}}: {val},".format,
-    }
-
-
 def pretty_repr(src, indent=0, no_indent_start=False, max_indent=20):
-    """Make human readable repr of object
+    warnings.warn(
-
+        'helpers.decorators.pretty_repr is deprecated'
-    :param src: object to process
+        ' in favor of external logwrap',
-    :type src: object
+        DeprecationWarning
-    :param indent: start indentation, all next levels is +4
+    )
-    :type indent: int
+    return ext_logwrap.pretty_repr(
-    :param no_indent_start: do not indent open bracket and simple parameters
+        src=src,
-    :type no_indent_start: bool
+        indent=indent,
-    :param max_indent: maximal indent before classic repr() call
+        no_indent_start=no_indent_start,
-    :type max_indent: int
+        max_indent=max_indent
-    :return: formatted string
-    """
-    if _simple(src) or indent >= max_indent:
-        indent = 0 if no_indent_start else indent
-        if isinstance(src, (six.binary_type, six.text_type)):
-            if isinstance(src, six.binary_type):
-                string = src.decode(
-                    encoding='utf-8',
-                    errors='backslashreplace'
-                )
-                prefix = 'b'
-            else:
-                string = src
-                prefix = 'u'
-            return _formatters['text'](
-                spc='',
-                indent=indent,
-                prefix=prefix,
-                string=string
-            )
-        return _formatters['simple'](
-            spc='',
-            indent=indent,
-            val=src
-        )
-    if isinstance(src, dict):
-        prefix, suffix = '{', '}'
-        result = ''
-        max_len = len(max([repr(key) for key in src])) if src else 0
-        for key, val in src.items():
-            result += _formatters['dict'](
-                spc='',
-                indent=indent + 4,
-                size=max_len,
-                key=key,
-                val=pretty_repr(val, indent + 8, no_indent_start=True)
-            )
-        return (
-            '\n{start:>{indent}}'.format(
-                start=prefix,
-                indent=indent + 1
-            ) +
-            result +
-            '\n{end:>{indent}}'.format(end=suffix, indent=indent + 1)
-        )
-    if isinstance(src, list):
-        prefix, suffix = '[', ']'
-    elif isinstance(src, tuple):
-        prefix, suffix = '(', ')'
-    else:
-        prefix, suffix = '{', '}'
-    result = ''
-    for elem in src:
-        if _simple(elem):
-            result += '\n'
-        result += pretty_repr(elem, indent + 4) + ','
-    return (
-        '\n{start:>{indent}}'.format(
-            start=prefix,
-            indent=indent + 1) +
-        result +
-        '\n{end:>{indent}}'.format(end=suffix, indent=indent + 1)
     )
 
 
 def logwrap(log=logger, log_level=logging.DEBUG, exc_level=logging.ERROR):
-    """Log function calls
+    warnings.warn(
-
+        'helpers.decorators.logwrap is deprecated'
-    :type log: logging.Logger
+        ' in favor of external logwrap',
-    :type log_level: int
+        DeprecationWarning
-    :type exc_level: int
+    )
-    :rtype: callable
+    return ext_logwrap.logwrap(
-    """
+        func=log if callable(log) else None,
-    def real_decorator(func):
+        log=log if isinstance(log, logging.Logger) else logger,
-        @functools.wraps(func)
+        log_level=log_level,
-        def wrapped(*args, **kwargs):
+        exc_level=exc_level
-            call_args = _getcallargs(func, *args, **kwargs)
+    )
-            args_repr = ""
-            if len(call_args) > 0:
-                args_repr = "\n    " + "\n    ".join((
-                    "{key!r}={val},".format(
-                        key=key,
-                        val=pretty_repr(val, indent=8, no_indent_start=True)
-                    )
-                    for key, val in call_args.items())
-                ) + '\n'
-            log.log(
-                level=log_level,
-                msg="Calling: \n{name!r}({arguments})".format(
-                    name=func.__name__,
-                    arguments=args_repr
-                )
-            )
-            try:
-                result = func(*args, **kwargs)
-                log.log(
-                    level=log_level,
-                    msg="Done: {name!r} with result:\n{result}".format(
-                        name=func.__name__,
-                        result=pretty_repr(result))
-                )
-            except BaseException:
-                log.log(
-                    level=exc_level,
-                    msg="Failed: \n{name!r}({arguments})".format(
-                        name=func.__name__,
-                        arguments=args_repr,
-                    ),
-                    exc_info=True
-                )
-                raise
-            return result
-        return wrapped
-
-    if not isinstance(log, logging.Logger):
-        func, log = log, logger
-        return real_decorator(func)
-
-    return real_decorator
 
 
 def proc_lock(path=settings.DEVOPS_LOCK_FILE, timeout=300):

devops/helpers/exec_result.py

@@ -12,369 +12,17 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from __future__ import absolute_import
 from __future__ import unicode_literals
 
-import json
+import warnings
-import threading
 
-import yaml
+from exec_helpers import ExecResult
 
-from devops import error
+__all__ = ('ExecResult',)
-from devops.helpers import proc_enums
-from devops import logger
 
-
+warnings.warn(
-deprecated_aliases = {
+    'devops.helpers.exec_result.ExecResult is deprecated'
-    'stdout_str',
+    ' in favor of external exec_helpers',
-    'stderr_str',
+    DeprecationWarning
-    'stdout_json',
+)
-    'stdout_yaml'
-}
-
-
-class ExecResult(object):
-    __slots__ = [
-        '__cmd', '__stdout', '__stderr', '__exit_code',
-        '__stdout_str', '__stderr_str', '__stdout_brief', '__stderr_brief',
-        '__stdout_json', '__stdout_yaml',
-        '__lock'
-    ]
-
-    def __init__(self, cmd, stdout=None, stderr=None,
-                 exit_code=proc_enums.ExitCodes.EX_INVALID):
-        """Command execution result read from fifo
-
-        :type cmd: str
-        :type stdout: list
-        :type stderr: list
-        :type exit_code: ExitCodes
-        """
-        self.__lock = threading.RLock()
-
-        self.__cmd = cmd
-        self.__stdout = stdout if stdout is not None else []
-        self.__stderr = stderr if stderr is not None else []
-
-        self.__exit_code = None
-        self.exit_code = exit_code
-
-        # By default is none:
-        self.__stdout_str = None
-        self.__stderr_str = None
-        self.__stdout_brief = None
-        self.__stderr_brief = None
-
-        self.__stdout_json = None
-        self.__stdout_yaml = None
-
-    @property
-    def lock(self):
-        """Lock object for thread-safe operation
-
-        :rtype: RLock
-        """
-        return self.__lock
-
-    @staticmethod
-    def _get_bytearray_from_array(src):
-        """Get bytearray from array of bytes blocks
-
-        :type src: list(bytes)
-        :rtype: bytearray
-        """
-        return bytearray(b''.join(src))
-
-    @staticmethod
-    def _get_str_from_bin(src):
-        """Join data in list to the string, with python 2&3 compatibility.
-
-        :type src: bytearray
-        :rtype: str
-        """
-        return src.strip().decode(
-            encoding='utf-8',
-            errors='backslashreplace'
-        )
-
-    @classmethod
-    def _get_brief(cls, data):
-        """Get brief output: 7 lines maximum (3 first + ... + 3 last)
-
-        :type data: list(bytes)
-        :rtype: str
-        """
-        src = data if len(data) <= 7 else data[:3] + [b'...\n'] + data[-3:]
-        return cls._get_str_from_bin(
-            cls._get_bytearray_from_array(src)
-        )
-
-    @property
-    def cmd(self):
-        """Executed command
-
-        :rtype: str
-        """
-        return self.__cmd
-
-    @property
-    def stdout(self):
-        """Stdout output as list of binaries
-
-        :rtype: list(bytes)
-        """
-        return self.__stdout
-
-    @stdout.setter
-    def stdout(self, new_val):
-        """Stdout output as list of binaries
-
-        :type new_val: list(bytes)
-        :raises: TypeError
-        """
-        if not isinstance(new_val, (list, type(None))):
-            raise TypeError('stdout should be list only!')
-        with self.lock:
-            self.__stdout_str = None
-            self.__stdout_brief = None
-            self.__stdout_json = None
-            self.__stdout_yaml = None
-            self.__stdout = new_val
-
-    @property
-    def stderr(self):
-        """Stderr output as list of binaries
-
-        :rtype: list(bytes)
-        """
-        return self.__stderr
-
-    @stderr.setter
-    def stderr(self, new_val):
-        """Stderr output as list of binaries
-
-        :type new_val: list(bytes)
-        :raises: TypeError
-        """
-        if not isinstance(new_val, (list, None)):
-            raise TypeError('stderr should be list only!')
-        with self.lock:
-            self.__stderr_str = None
-            self.__stderr_brief = None
-            self.__stderr = new_val
-
-    @property
-    def stdout_bin(self):
-        """Stdout in binary format
-
-        Sometimes logging is used to log binary objects too (example: Session),
-        and for debug purposes we can use this as data source.
-        :rtype: bytearray
-        """
-        with self.lock:
-            return self._get_bytearray_from_array(self.stdout)
-
-    @property
-    def stderr_bin(self):
-        """Stderr in binary format
-
-        :rtype: bytearray
-        """
-        with self.lock:
-            return self._get_bytearray_from_array(self.stderr)
-
-    @property
-    def stdout_str(self):
-        """Stdout output as string
-
-        :rtype: str
-        """
-        with self.lock:
-            if self.__stdout_str is None:
-                self.__stdout_str = self._get_str_from_bin(self.stdout_bin)
-            return self.__stdout_str
-
-    @property
-    def stderr_str(self):
-        """Stderr output as string
-
-        :rtype: str
-        """
-        with self.lock:
-            if self.__stderr_str is None:
-                self.__stderr_str = self._get_str_from_bin(self.stderr_bin)
-            return self.__stderr_str
-
-    @property
-    def stdout_brief(self):
-        """Brief stdout output (mostly for exceptions)
-
-        :rtype: str
-        """
-        with self.lock:
-            if self.__stdout_brief is None:
-                self.__stdout_brief = self._get_brief(self.stdout)
-            return self.__stdout_brief
-
-    @property
-    def stderr_brief(self):
-        """Brief stderr output (mostly for exceptions)
-
-        :rtype: str
-        """
-        with self.lock:
-            if self.__stderr_brief is None:
-                self.__stderr_brief = self._get_brief(self.stderr)
-            return self.__stderr_brief
-
-    @property
-    def exit_code(self):
-        """Return(exit) code of command
-
-        :rtype: int
-        """
-        return self.__exit_code
-
-    @exit_code.setter
-    def exit_code(self, new_val):
-        """Return(exit) code of command
-
-        :type new_val: int
-        """
-        if not isinstance(new_val, (int, proc_enums.ExitCodes)):
-            raise TypeError('Exit code is strictly int')
-        with self.lock:
-            if isinstance(new_val, int) and \
-                    new_val in proc_enums.ExitCodes.__members__.values():
-                new_val = proc_enums.ExitCodes(new_val)
-            self.__exit_code = new_val
-
-    def __deserialize(self, fmt):
-        """Deserialize stdout as data format
-
-        :type fmt: str
-        :rtype: object
-        :raises: DevopsError
-        """
-        try:
-            if fmt == 'json':
-                return json.loads(self.stdout_str, encoding='utf-8')
-            elif fmt == 'yaml':
-                return yaml.safe_load(self.stdout_str)
-        except BaseException:
-            tmpl = (
-                " stdout is not valid {fmt}:\n"
-                '{{stdout!r}}\n'.format(
-                    fmt=fmt))
-            logger.exception(self.cmd + tmpl.format(stdout=self.stdout_str))
-            raise error.DevopsError(
-                self.cmd + tmpl.format(stdout=self.stdout_brief))
-        msg = '{fmt} deserialize target is not implemented'.format(fmt=fmt)
-        logger.error(msg)
-        raise error.DevopsNotImplementedError(msg)
-
-    @property
-    def stdout_json(self):
-        """JSON from stdout
-
-        :rtype: object
-        """
-        with self.lock:
-            if self.__stdout_json is None:
-                # noinspection PyTypeChecker
-                self.__stdout_json = self.__deserialize(fmt='json')
-            return self.__stdout_json
-
-    @property
-    def stdout_yaml(self):
-        """YAML from stdout
-
-        :rtype: Union(list, dict, None)
-        """
-        with self.lock:
-            if self.__stdout_yaml is None:
-                # noinspection PyTypeChecker
-                self.__stdout_yaml = self.__deserialize(fmt='yaml')
-            return self.__stdout_yaml
-
-    def __dir__(self):
-        return [
-            'cmd', 'stdout', 'stderr', 'exit_code',
-            'stdout_bin', 'stderr_bin',
-            'stdout_str', 'stderr_str', 'stdout_brief', 'stderr_brief',
-            'stdout_json', 'stdout_yaml',
-            'lock'
-        ]
-
-    def __getitem__(self, item):
-        if item in dir(self):
-            return getattr(self, item)
-        raise IndexError(
-            '"{item}" not found in {dir}'.format(
-                item=item, dir=dir(self)
-            )
-        )
-
-    def __setitem__(self, key, value):
-        rw = ['stdout', 'stderr', 'exit_code']
-        if key in rw:
-            setattr(self, key, value)
-            return
-        if key in deprecated_aliases:
-            logger.warning(
-                '{key} is read-only and calculated automatically'.format(
-                    key=key
-                )
-            )
-            return
-        if key in dir(self):
-            raise error.DevopsError(
-                '{key} is read-only!'.format(key=key)
-            )
-        raise IndexError(
-            '{key} not found in {dir}'.format(
-                key=key, dir=rw
-            )
-        )
-
-    def __repr__(self):
-        return (
-            '{cls}(cmd={cmd!r}, stdout={stdout}, stderr={stderr}, '
-            'exit_code={exit_code!s})'.format(
-                cls=self.__class__.__name__,
-                cmd=self.cmd,
-                stdout=self.stdout,
-                stderr=self.stderr,
-                exit_code=self.exit_code
-            ))
-
-    def __str__(self):
-        return (
-            "{cls}(\n\tcmd={cmd!r},"
-            "\n\t stdout=\n'{stdout_brief}',"
-            "\n\tstderr=\n'{stderr_brief}', "
-            '\n\texit_code={exit_code!s}\n)'.format(
-                cls=self.__class__.__name__,
-                cmd=self.cmd,
-                stdout_brief=self.stdout_brief,
-                stderr_brief=self.stderr_brief,
-                exit_code=self.exit_code
-            )
-        )
-
-    def __eq__(self, other):
-        return all(
-            (
-                getattr(self, val) == getattr(other, val)
-                for val in ['cmd', 'stdout', 'stderr', 'exit_code']
-            )
-        )
-
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
-    def __hash__(self):
-        return hash(
-            (
-                self.__class__, self.cmd, self.stdout_str, self.stderr_str,
-                self.exit_code
-            ))

devops/helpers/helpers.py

@@ -25,6 +25,7 @@ import warnings
 import xml.etree.ElementTree as ET
 
 from dateutil import tz
+import exec_helpers
 import six
 # pylint: disable=import-error
 # noinspection PyUnresolvedReferences
@@ -34,8 +35,6 @@ from six.moves import xmlrpc_client
 # pylint: enable=import-error
 
 from devops import error
-from devops.helpers import ssh_client
-from devops.helpers import subprocess_runner
 from devops import logger
 from devops import settings
 
@@ -53,7 +52,7 @@ def icmp_ping(host, timeout=1):
     returns True if host is pingable
     False - otherwise.
     """
-    result = subprocess_runner.Subprocess.execute(
+    result = exec_helpers.Subprocess().execute(
         "ping -c 1 -W '{timeout:d}' '{host:s}'".format(
             host=host, timeout=timeout))
     return result.exit_code == 0
@@ -238,10 +237,10 @@ def wait_ssh_cmd(
         username=settings.SSH_CREDENTIALS['login'],
         password=settings.SSH_CREDENTIALS['password'],
         timeout=0):
-    ssh = ssh_client.SSHClient(host=host, port=port,
+    ssh = exec_helpers.SSHClient(host=host, port=port,
-                               auth=ssh_client.SSHAuth(
+                                 auth=exec_helpers.SSHAuth(
-                                   username=username,
+                                     username=username,
-                                   password=password))
+                                     password=password))
     wait(lambda: not ssh.execute(check_cmd)['exit_code'],
          timeout=timeout)
 

devops/helpers/log_templates.py

@@ -1,26 +0,0 @@
-# coding=utf-8
-
-#    Copyright 2017 Mirantis, Inc.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-CMD_EXEC = u"Executing command:\n{cmd!s}\n"
-CMD_RESULT = (u"Command exit code '{code!s}':\n{cmd!s}\n")
-CMD_UNEXPECTED_EXIT_CODE = (u"{append}Command '{cmd!s}' returned "
-                            u"exit code '{code!s}' while "
-                            u"expected '{expected!s}'\n")
-CMD_UNEXPECTED_STDERR = (u"{append}Command '{cmd!s}' STDERR while "
-                         u"not expected\n"
-                         u"\texit code: '{code!s}'")
-CMD_WAIT_ERROR = (u"Wait for '{cmd!s}' during {timeout!s}s: "
-                  u"no return code!")

devops/helpers/proc_enums.py

@@ -12,113 +12,17 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import enum
+from __future__ import absolute_import
+from __future__ import unicode_literals
 
+import warnings
 
-@enum.unique
+from exec_helpers import ExitCodes
-class SigNum(enum.IntEnum):
-    SIGHUP = 1  # Hangup (POSIX).
-    SIGINT = 2  # Interrupt (ANSI).
-    SIGQUIT = 3  # Quit (POSIX).
-    SIGILL = 4  # Illegal instruction (ANSI).
-    SIGTRAP = 5  # Trace trap (POSIX).
-    SIGABRT = 6  # Abort (ANSI).
-    SIGBUS = 7  # BUS error (4.2 BSD).
-    SIGFPE = 8  # Floating-point exception (ANSI).
-    SIGKILL = 9  # Kill, unblockable (POSIX).
-    SIGUSR1 = 10  # User-defined signal 1 (POSIX).
-    SIGSEGV = 11  # Segmentation violation (ANSI).
-    SIGUSR2 = 12  # User-defined signal 2 (POSIX).
-    SIGPIPE = 13  # Broken pipe (POSIX).
-    SIGALRM = 14  # Alarm clock (POSIX).
-    SIGTERM = 15  # Termination (ANSI).
-    SIGSTKFLT = 16  # Stack fault.
-    SIGCHLD = 17  # Child status has changed (POSIX).
-    SIGCONT = 18  # Continue (POSIX).
-    SIGSTOP = 19  # Stop, unblockable (POSIX).
-    SIGTSTP = 20  # Keyboard stop (POSIX).
-    SIGTTIN = 21  # Background read from tty (POSIX).
-    SIGTTOU = 22  # Background write to tty (POSIX).
-    SIGURG = 23  # Urgent condition on socket (4.2 BSD).
-    SIGXCPU = 24  # CPU limit exceeded (4.2 BSD).
-    SIGXFSZ = 25  # File size limit exceeded (4.2 BSD).
-    SIGVTALRM = 26  # Virtual alarm clock (4.2 BSD).
-    SIGPROF = 27  # Profiling alarm clock (4.2 BSD).
-    SIGWINCH = 28  # Window size change (4.3 BSD, Sun).
-    SIGPOLL = 29  # Pollable event occurred (System V)
-    SIGPWR = 30  # Power failure restart (System V).
-    SIGSYS = 31  # Bad system call.
 
-    def __str__(self):
+__all__ = ('ExitCodes',)
-        return "{name}<{value:d}(0x{value:02X})>".format(
-            name=self.name,
-            value=self.value
-        )
 
-
+warnings.warn(
-@enum.unique
+    'devops.helpers.proc_enums.ExitCodes is deprecated'
-class ExitCodes(enum.IntEnum):
+    ' in favor of external exec_helpers',
-    EX_OK = 0  # successful termination
+    DeprecationWarning
-
+)
-    EX_INVALID = 0xDEADBEEF  # uint32 debug value. Impossible for POSIX
-
-    EX_ERROR = 1  # general failure
-    EX_BUILTIN = 2  # Misuse of shell builtins (according to Bash)
-
-    EX_USAGE = 64  # command line usage error
-    EX_DATAERR = 65  # data format error
-    EX_NOINPUT = 66  # cannot open input
-    EX_NOUSER = 67  # addressee unknown
-    EX_NOHOST = 68  # host name unknown
-    EX_UNAVAILABLE = 69  # service unavailable
-    EX_SOFTWARE = 70  # internal software error
-    EX_OSERR = 71  # system error (e.g., can't fork)
-    EX_OSFILE = 72  # critical OS file missing
-    EX_CANTCREAT = 73  # can't create (user) output file
-    EX_IOERR = 74  # input/output error
-    EX_TEMPFAIL = 75  # temp failure; user is invited to retry
-    EX_PROTOCOL = 76  # remote error in protocol
-    EX_NOPERM = 77  # permission denied
-    EX_CONFIG = 78  # configuration error
-
-    EX_NOEXEC = 126  # If a command is found but is not executable
-    EX_NOCMD = 127  # If a command is not found
-
-    # Signal exits:
-    EX_SIGHUP = 128 + SigNum.SIGHUP
-    EX_SIGINT = 128 + SigNum.SIGINT
-    EX_SIGQUIT = 128 + SigNum.SIGQUIT
-    EX_SIGILL = 128 + SigNum.SIGILL
-    EX_SIGTRAP = 128 + SigNum.SIGTRAP
-    EX_SIGABRT = 128 + SigNum.SIGABRT
-    EX_SIGBUS = 128 + SigNum.SIGBUS
-    EX_SIGFPE = 128 + SigNum.SIGFPE
-    EX_SIGKILL = 128 + SigNum.SIGKILL
-    EX_SIGUSR1 = 128 + SigNum.SIGUSR1
-    EX_SIGSEGV = 128 + SigNum.SIGSEGV
-    EX_SIGUSR2 = 128 + SigNum.SIGUSR2
-    EX_SIGPIPE = 128 + SigNum.SIGPIPE
-    EX_SIGALRM = 128 + SigNum.SIGALRM
-    EX_SIGTERM = 128 + SigNum.SIGTERM
-    EX_SIGSTKFLT = 128 + SigNum.SIGSTKFLT
-    EX_SIGCHLD = 128 + SigNum.SIGCHLD
-    EX_SIGCONT = 128 + SigNum.SIGCONT
-    EX_SIGSTOP = 128 + SigNum.SIGSTOP
-    EX_SIGTSTP = 128 + SigNum.SIGTSTP
-    EX_SIGTTIN = 128 + SigNum.SIGTTIN
-    EX_SIGTTOU = 128 + SigNum.SIGTTOU
-    EX_SIGURG = 128 + SigNum.SIGURG
-    EX_SIGXCPU = 128 + SigNum.SIGXCPU
-    EX_SIGXFSZ = 128 + SigNum.SIGXFSZ
-    EX_SIGVTALRM = 128 + SigNum.SIGVTALRM
-    EX_SIGPROF = 128 + SigNum.SIGPROF
-    EX_SIGWINCH = 128 + SigNum.SIGWINCH
-    EX_SIGPOLL = 128 + SigNum.SIGPOLL
-    EX_SIGPWR = 128 + SigNum.SIGPWR
-    EX_SIGSYS = 128 + SigNum.SIGSYS
-
-    def __str__(self):
-        return "{name}<{value:d}(0x{value:02X})>".format(
-            name=self.name,
-            value=self.value
-        )

devops/helpers/retry.py

@@ -1,25 +0,0 @@
-#    Copyright 2013 - 2016 Mirantis, Inc.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import warnings
-
-from devops.helpers.decorators import retry
-from devops import logger
-
-
-msg = 'devops.helpers.retry.retry was moved to devops.helpers.decorators.retry'
-logger.critical(msg)
-warnings.warn(msg, DeprecationWarning)
-
-__all__ = ['retry']

devops/helpers/subprocess_runner.py

@@ -12,275 +12,18 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from __future__ import absolute_import
 from __future__ import unicode_literals
 
-import fcntl
+import warnings
-import os
-import select
-import subprocess
-import threading
-import time
 
-import six
+from exec_helpers import Subprocess
-
-from devops import error
-from devops.helpers import decorators
-from devops.helpers import exec_result
-from devops.helpers import log_templates
-from devops.helpers import metaclasses
-from devops.helpers import proc_enums
-from devops import logger
 
 
-class Subprocess(six.with_metaclass(metaclasses.SingletonMeta, object)):
+__all__ = ('Subprocess',)
-    __lock = threading.RLock()
 
-    def __init__(self):
+warnings.warn(
-        """Subprocess helper with timeouts and lock-free FIFO
+    'helpers.subprocess_runner.Subprocess is deprecated '
-
+    'in favor of external exec_helpers.Subprocess',
-        For excluding race-conditions we allow to run 1 command simultaneously
+    DeprecationWarning
-        """
+)
-        pass
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        pass
-
-    @classmethod
-    def __exec_command(cls, command, cwd=None, env=None, timeout=None,
-                       verbose=False):
-        """Command executor helper
-
-        :type command: str
-        :type cwd: str
-        :type env: dict
-        :type timeout: int
-        :rtype: ExecResult
-        """
-        def poll_stream(src, verb_logger=None):
-            dst = []
-            try:
-                for line in src:
-                    dst.append(line)
-                    if verb_logger is not None:
-                        verb_logger(
-                            line.decode('utf-8',
-                                        errors='backslashreplace').rstrip()
-                        )
-            except IOError:
-                pass
-            return dst
-
-        def poll_streams(result, stdout, stderr, verbose):
-            rlist, _, _ = select.select(
-                [stdout, stderr],
-                [],
-                [])
-            if rlist:
-                if stdout in rlist:
-                    result.stdout += poll_stream(
-                        src=stdout,
-                        verb_logger=logger.info if verbose else logger.debug)
-                if stderr in rlist:
-                    result.stderr += poll_stream(
-                        src=stderr,
-                        verb_logger=logger.error if verbose else logger.debug)
-
-        @decorators.threaded(started=True)
-        def poll_pipes(proc, result, stop):
-            """Polling task for FIFO buffers
-
-            :type proc: subprocess.Popen
-            :type result: ExecResult
-            :type stop: threading.Event
-            """
-            # Get file descriptors for stdout and stderr streams
-            fd_stdout = proc.stdout.fileno()
-            fd_stderr = proc.stderr.fileno()
-            # Get flags of stdout and stderr streams
-            fl_stdout = fcntl.fcntl(fd_stdout, fcntl.F_GETFL)
-            fl_stderr = fcntl.fcntl(fd_stderr, fcntl.F_GETFL)
-            # Set nonblock mode for stdout and stderr streams
-            fcntl.fcntl(fd_stdout, fcntl.F_SETFL, fl_stdout | os.O_NONBLOCK)
-            fcntl.fcntl(fd_stderr, fcntl.F_SETFL, fl_stderr | os.O_NONBLOCK)
-
-            while not stop.isSet():
-                time.sleep(0.1)
-                poll_streams(
-                    result=result,
-                    stdout=proc.stdout,
-                    stderr=proc.stderr,
-                    verbose=verbose
-                )
-
-                proc.poll()
-
-                if proc.returncode is not None:
-                    result.exit_code = proc.returncode
-                    result.stdout += poll_stream(
-                        src=proc.stdout,
-                        verb_logger=logger.info if verbose else logger.debug)
-                    result.stderr += poll_stream(
-                        src=proc.stderr,
-                        verb_logger=logger.error if verbose else logger.debug)
-
-                    stop.set()
-
-        # 1 Command per run
-        with cls.__lock:
-            result = exec_result.ExecResult(cmd=command)
-            stop_event = threading.Event()
-            message = log_templates.CMD_EXEC.format(cmd=command.rstrip())
-            if verbose:
-                logger.info(message)
-            else:
-                logger.debug(message)
-            # Run
-            process = subprocess.Popen(
-                args=[command],
-                stdin=subprocess.PIPE,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                shell=True, cwd=cwd, env=env,
-                universal_newlines=False)
-
-            # Poll output
-            poll_pipes(process, result, stop_event)
-            # wait for process close
-            stop_event.wait(timeout)
-
-            # Process closed?
-            if stop_event.isSet():
-                stop_event.clear()
-                return result
-            # Kill not ended process and wait for close
-            try:
-                process.kill()  # kill -9
-                stop_event.wait(5)
-
-            except OSError:
-                # Nothing to kill
-                logger.warning(
-                    u"{!s} has been completed just after timeout: "
-                    "please validate timeout.".format(command))
-
-            wait_err_msg = log_templates.CMD_WAIT_ERROR.format(
-                cmd=command.rstrip(),
-                timeout=timeout)
-            output_brief_msg = ('\tSTDOUT:\n'
-                                '{0}\n'
-                                '\tSTDERR"\n'
-                                '{1}'.format(result.stdout_brief,
-                                             result.stderr_brief))
-            logger.debug(wait_err_msg)
-            raise error.TimeoutError(wait_err_msg + output_brief_msg)
-
-    @classmethod
-    def execute(cls, command, verbose=False, timeout=None, **kwargs):
-        """Execute command and wait for return code
-
-        Timeout limitation: read tick is 100 ms.
-
-        :type command: str
-        :type verbose: bool
-        :type timeout: int
-        :rtype: ExecResult
-        :raises: TimeoutError
-        """
-        result = cls.__exec_command(command=command, timeout=timeout,
-                                    verbose=verbose, **kwargs)
-        message = log_templates.CMD_RESULT.format(
-            cmd=command, code=result.exit_code)
-        if verbose:
-            logger.info(message)
-        else:
-            logger.debug(message)
-
-        return result
-
-    @classmethod
-    def check_call(
-            cls,
-            command, verbose=False, timeout=None,
-            error_info=None,
-            expected=None, raise_on_err=True, **kwargs):
-        """Execute command and check for return code
-
-        Timeout limitation: read tick is 100 ms.
-
-        :type command: str
-        :type verbose: bool
-        :type timeout: int
-        :type error_info: str
-        :type expected: list
-        :type raise_on_err: bool
-        :rtype: ExecResult
-        :raises: DevopsCalledProcessError
-        """
-
-        if expected is None:
-            expected = [proc_enums.ExitCodes.EX_OK]
-        else:
-            expected = [
-                proc_enums.ExitCodes(code)
-                if (
-                    isinstance(code, int) and
-                    code in proc_enums.ExitCodes.__members__.values())
-                else code
-                for code in expected
-                ]
-        ret = cls.execute(command, verbose, timeout, **kwargs)
-        if ret['exit_code'] not in expected:
-            message = (
-                log_templates.CMD_UNEXPECTED_EXIT_CODE.format(
-                    append=error_info + '\n' if error_info else '',
-                    cmd=command,
-                    code=ret['exit_code'],
-                    expected=expected
-                ))
-            logger.error(message)
-            if raise_on_err:
-                raise error.DevopsCalledProcessError(
-                    command, ret['exit_code'],
-                    expected=expected,
-                    stdout=ret['stdout_brief'],
-                    stderr=ret['stderr_brief'])
-        return ret
-
-    @classmethod
-    def check_stderr(
-            cls,
-            command, verbose=False, timeout=None,
-            error_info=None,
-            raise_on_err=True, **kwargs):
-        """Execute command expecting return code 0 and empty STDERR
-
-        Timeout limitation: read tick is 100 ms.
-
-        :type command: str
-        :type verbose: bool
-        :type timeout: int
-        :type error_info: str
-        :type raise_on_err: bool
-        :rtype: ExecResult
-        :raises: DevopsCalledProcessError
-        """
-        ret = cls.check_call(
-            command, verbose, timeout=timeout,
-            error_info=error_info, raise_on_err=raise_on_err, **kwargs)
-        if ret['stderr']:
-            message = (
-                log_templates.CMD_UNEXPECTED_STDERR.format(
-                    append=error_info + '\n' if error_info else '',
-                    cmd=command,
-                    code=ret['exit_code']
-                ))
-            logger.error(message)
-            if raise_on_err:
-                raise error.DevopsCalledProcessError(
-                    command, ret['exit_code'],
-                    stdout=ret['stdout_brief'],
-                    stderr=ret['stderr_brief'])
-        return ret

devops/models/environment.py

@@ -18,13 +18,13 @@ import warnings
 from django.conf import settings
 from django.db import IntegrityError
 from django.db import models
+import exec_helpers
 import netaddr
 import paramiko
 
 from devops import error
 from devops.helpers import decorators
 from devops.helpers import network as network_helpers
-from devops.helpers import ssh_client
 from devops import logger
 from devops.models import base
 from devops.models import driver
@@ -417,9 +417,9 @@ class Environment(base.BaseModel):
 
         from devops import client
         env = client.DevopsClient().get_env(self.name)
-        return ssh_client.SSHClient(
+        return exec_helpers.SSHClient(
             ip,
-            auth=ssh_client.SSHAuth(
+            auth=exec_helpers.SSHAuth(
                 username=login, password=password,
                 keys=env.get_private_keys()))
 
@@ -435,9 +435,9 @@ class Environment(base.BaseModel):
             logger.warning('Loading of SSH key from file failed. Trying to use'
                            ' SSH agent ...')
             keys = paramiko.Agent().get_keys()
-        return ssh_client.SSHClient(
+        return exec_helpers.SSHClient(
             ip,
-            auth=ssh_client.SSHAuth(keys=keys))
+            auth=exec_helpers.SSHAuth(keys=keys))
 
     # LEGACY, TO REMOVE (for fuel-qa compatibility)
     def nodes(self):  # migrated from EnvironmentModel.nodes()

devops/tests/client/test_environment.py

@@ -14,12 +14,13 @@
 
 import mock
 
+import exec_helpers
+
 from devops.client import environment
 from devops.client import nailgun
 from devops import error
 from devops.helpers import helpers
 from devops.helpers import ntp
-from devops.helpers import ssh_client
 from devops.tests.driver import driverless
 
 
@@ -41,7 +42,7 @@ class TestDevopsEnvironment(driverless.DriverlessTestCase):
         self.wait_tcp_mock = self.patch(
             'devops.helpers.helpers.wait_tcp', spec=helpers.wait_tcp)
         self.ssh_mock = self.patch(
-            'devops.helpers.ssh_client.SSHClient', spec=ssh_client.SSHClient)
+            'exec_helpers.SSHClient', spec=exec_helpers.SSHClient)
         self.nc_mock = self.patch(
             'devops.client.nailgun.NailgunClient', spec=nailgun.NailgunClient)
         self.nc_mock_inst = self.nc_mock.return_value
@@ -172,7 +173,7 @@ class TestDevopsEnvironment(driverless.DriverlessTestCase):
         assert remote is ssh
         self.ssh_mock.assert_called_once_with(
             '10.109.0.2',
-            auth=ssh_client.SSHAuth(username='root', password='r00tme'))
+            auth=exec_helpers.SSHAuth(username='root', password='r00tme'))
 
         self.wait_tcp_mock.assert_called_once_with(
             host='10.109.0.2', port=22, timeout=180,
@@ -224,7 +225,7 @@ class TestDevopsEnvironment(driverless.DriverlessTestCase):
 
         self.ssh_mock.assert_called_once_with(
             '10.109.0.2',
-            auth=ssh_client.SSHAuth(username='root', password='r00tme'))
+            auth=exec_helpers.SSHAuth(username='root', password='r00tme'))
         assert ssh.isfile.call_count == 2
         ssh.isfile.assert_any_call('/root/.ssh/id_rsa')
         ssh.isfile.assert_any_call('/root/.ssh/bootstrap.rsa')
@@ -268,7 +269,7 @@ class TestDevopsEnvironment(driverless.DriverlessTestCase):
         assert remote is ssh
         self.ssh_mock.assert_called_with(
             '10.109.0.100',
-            auth=ssh_client.SSHAuth(
+            auth=exec_helpers.SSHAuth(
                 username='root',
                 password='r00tme',
                 keys=keys))

devops/tests/helpers/test_cloud_image_settings.py

@@ -32,7 +32,7 @@ class TestCloudImageSettings(unittest.TestCase):
 
     def setUp(self):
         self.subprocess_mock = self.patch(
-            'devops.helpers.subprocess_runner.Subprocess', autospec=True)
+            'exec_helpers.Subprocess', autospec=True)
 
         self.os_mock = self.patch(
             'devops.helpers.cloud_image_settings.os', autospec=True)
@@ -93,7 +93,7 @@ class TestCloudImageSettings(unittest.TestCase):
             mock.call().__exit__(None, None, None),
         ))
 
-        self.subprocess_mock.check_call.assert_called_once_with(
+        self.subprocess_mock().check_call.assert_called_once_with(
             'genisoimage -output /mydir/cloud_settings.iso '
             '-volid cidata -joliet -rock /mydir/user-data '
             '/mydir/meta-data')

devops/tests/helpers/test_decorators.py

@@ -14,8 +14,6 @@
 
 from __future__ import unicode_literals
 
-import logging
-import threading
 import unittest
 
 import mock
@@ -24,97 +22,6 @@ from devops import error
 from devops.helpers import decorators
 
 
-class ThreadedTest(unittest.TestCase):
-    def test_add_basic(self):
-        @decorators.threaded
-        def func_test():
-            pass
-        # pylint: disable=assignment-from-no-return
-        test_thread = func_test()
-        # pylint: enable=assignment-from-no-return
-        self.assertEqual(test_thread.name, 'Threaded func_test')
-        self.assertFalse(test_thread.daemon)
-        self.assertFalse(test_thread.isAlive())
-
-    def test_add_func(self):
-        @decorators.threaded()
-        def func_test():
-            pass
-
-        # pylint: disable=assignment-from-no-return
-        test_thread = func_test()
-        # pylint: enable=assignment-from-no-return
-        self.assertEqual(test_thread.name, 'Threaded func_test')
-        self.assertFalse(test_thread.daemon)
-        self.assertFalse(test_thread.isAlive())
-
-    def test_name(self):
-        @decorators.threaded(name='test name')
-        def func_test():
-            pass
-
-        # pylint: disable=assignment-from-no-return
-        test_thread = func_test()
-        # pylint: enable=assignment-from-no-return
-        self.assertEqual(test_thread.name, 'test name')
-        self.assertFalse(test_thread.daemon)
-        self.assertFalse(test_thread.isAlive())
-
-    def test_daemon(self):
-        @decorators.threaded(daemon=True)
-        def func_test():
-            pass
-
-        # pylint: disable=assignment-from-no-return
-        test_thread = func_test()
-        # pylint: enable=assignment-from-no-return
-        self.assertEqual(test_thread.name, 'Threaded func_test')
-        self.assertTrue(test_thread.daemon)
-        self.assertFalse(test_thread.isAlive())
-
-    @mock.patch('threading.Thread', autospec=True)
-    def test_started(self, thread):
-        @decorators.threaded(started=True)
-        def func_test():
-            pass
-
-        func_test()
-
-        self.assertIn(mock.call().start(), thread.mock_calls)
-
-    def test_args(self):
-        event = threading.Event()
-        data = []
-        # pylint: disable=global-variable-not-assigned
-        global data
-        # pylint: enable=global-variable-not-assigned
-
-        @decorators.threaded(started=True)
-        def func_test(add, evnt):
-            data.append(add)
-            evnt.set()
-
-        func_test(1, event)
-        event.wait(3)
-        self.assertEqual(data, [1])
-
-    def test_kwargs(self):
-        event = threading.Event()
-        data = []
-        # pylint: disable=global-variable-not-assigned
-        global data
-        # pylint: enable=global-variable-not-assigned
-
-        @decorators.threaded(started=True)
-        def func_test(add, evnt):
-            data.append(add)
-            evnt.set()
-
-        func_test(add=2, evnt=event)
-        event.wait(3)
-        self.assertEqual(data, [2])
-
-
 class TestRetry(unittest.TestCase):
 
     def patch(self, *args, **kwargs):
@@ -196,315 +103,6 @@ class TestRetry(unittest.TestCase):
             retry_dec('wrong')
 
 
-class TestPrettyRepr(unittest.TestCase):
-    def test_simple(self):
-        self.assertEqual(
-            decorators.pretty_repr(True), repr(True)
-        )
-
-    def test_text(self):
-        self.assertEqual(
-            decorators.pretty_repr('Unicode text'), "u'''Unicode text'''"
-        )
-        self.assertEqual(
-            decorators.pretty_repr(b'bytes text\x01'), "b'''bytes text\x01'''"
-        )
-
-    def test_iterable(self):
-        self.assertEqual(
-            decorators.pretty_repr([1, 2, 3]),
-            '\n[{nl:<5}1,{nl:<5}2,{nl:<5}3,\n]'.format(nl='\n')
-        )
-        self.assertEqual(
-            decorators.pretty_repr((1, 2, 3)),
-            '\n({nl:<5}1,{nl:<5}2,{nl:<5}3,\n)'.format(nl='\n')
-        )
-        res = decorators.pretty_repr({1, 2, 3})
-        self.assertTrue(
-            res.startswith('\n{') and res.endswith('\n}')
-        )
-
-    def test_dict(self):
-        self.assertEqual(
-            decorators.pretty_repr({1: 1, 2: 2, 33: 33}),
-            '\n{\n    1 : 1,\n    2 : 2,\n    33: 33,\n}'
-        )
-
-    def test_nested_dict(self):
-        test_obj = [
-            {
-                1:
-                    {
-                        2: 3
-                    },
-                4:
-                    {
-                        5: 6
-                    },
-            },
-            {
-                7: 8,
-                9: (10, 11)
-            },
-            (
-                12,
-                13,
-            ),
-            {14: {15: {16: {17: {18: {19: [20]}}}}}}
-        ]
-        exp_repr = (
-            '\n['
-            '\n    {'
-            '\n        1: '
-            '\n            {'
-            '\n                2: 3,'
-            '\n            },'
-            '\n        4: '
-            '\n            {'
-            '\n                5: 6,'
-            '\n            },'
-            '\n    },'
-            '\n    {'
-            '\n        9: '
-            '\n            ('
-            '\n                10,'
-            '\n                11,'
-            '\n            ),'
-            '\n        7: 8,'
-            '\n    },'
-            '\n    ('
-            '\n        12,'
-            '\n        13,'
-            '\n    ),'
-            '\n    {'
-            '\n        14: '
-            '\n            {'
-            '\n                15: {16: {17: {18: {19: [20]}}}},'
-            '\n            },'
-            '\n    },'
-            '\n]'
-        )
-        self.assertEqual(decorators.pretty_repr(test_obj), exp_repr)
-
-
-@mock.patch('devops.helpers.decorators.logger', autospec=True)
-class TestLogWrap(unittest.TestCase):
-    def test_no_args(self, logger):
-        @decorators.logwrap
-        def func():
-            return 'No args'
-
-        result = func()
-        self.assertEqual(result, 'No args')
-        logger.assert_has_calls((
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Calling: \n'func'()"
-            ),
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Done: 'func' with result:\n{}".format(
-                    decorators.pretty_repr(result))
-            ),
-        ))
-
-    def test_args_simple(self, logger):
-        arg = 'test arg'
-
-        @decorators.logwrap
-        def func(tst):
-            return tst
-
-        result = func(arg)
-        self.assertEqual(result, arg)
-        logger.assert_has_calls((
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Calling: \n'func'(\n    'tst'={},\n)".format(
-                    decorators.pretty_repr(arg, indent=8, no_indent_start=True)
-                )
-            ),
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Done: 'func' with result:\n{}".format(
-                    decorators.pretty_repr(result))
-            ),
-        ))
-
-    def test_args_defaults(self, logger):
-        arg = 'test arg'
-
-        @decorators.logwrap
-        def func(tst=arg):
-            return tst
-
-        result = func()
-        self.assertEqual(result, arg)
-        logger.assert_has_calls((
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Calling: \n'func'(\n    'tst'={},\n)".format(
-                    decorators.pretty_repr(arg, indent=8,
-                                           no_indent_start=True))
-            ),
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Done: 'func' with result:\n{}".format(
-                    decorators.pretty_repr(result))
-            ),
-        ))
-
-    def test_args_complex(self, logger):
-        string = 'string'
-        dictionary = {'key': 'dictionary'}
-
-        @decorators.logwrap
-        def func(param_string, param_dictionary):
-            return param_string, param_dictionary
-
-        result = func(string, dictionary)
-        self.assertEqual(result, (string, dictionary))
-        # raise ValueError(logger.mock_calls)
-        logger.assert_has_calls((
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Calling: \n'func'("
-                    "\n    'param_string'={string},"
-                    "\n    'param_dictionary'={dictionary},\n)".format(
-                        string=decorators.pretty_repr(
-                            string,
-                            indent=8, no_indent_start=True),
-                        dictionary=decorators.pretty_repr(
-                            dictionary,
-                            indent=8, no_indent_start=True)
-                    )
-            ),
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Done: 'func' with result:\n{}".format(
-                    decorators.pretty_repr(result))
-            ),
-        ))
-
-    def test_args_kwargs(self, logger):
-        targs = ['string1', 'string2']
-        tkwargs = {'key': 'tkwargs'}
-
-        @decorators.logwrap
-        def func(*args, **kwargs):
-            return tuple(args), kwargs
-
-        result = func(*targs, **tkwargs)
-        self.assertEqual(result, (tuple(targs), tkwargs))
-        # raise ValueError(logger.mock_calls)
-        logger.assert_has_calls((
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Calling: \n'func'("
-                    "\n    'args'={args},"
-                    "\n    'kwargs'={kwargs},\n)".format(
-                        args=decorators.pretty_repr(
-                            tuple(targs),
-                            indent=8, no_indent_start=True),
-                        kwargs=decorators.pretty_repr(
-                            tkwargs,
-                            indent=8, no_indent_start=True)
-                    )
-            ),
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Done: 'func' with result:\n{}".format(
-                    decorators.pretty_repr(result))
-            ),
-        ))
-
-    def test_renamed_args_kwargs(self, logger):
-        arg = 'arg'
-        targs = ['string1', 'string2']
-        tkwargs = {'key': 'tkwargs'}
-
-        @decorators.logwrap
-        def func(arg, *positional, **named):
-            return arg, tuple(positional), named
-
-        result = func(arg, *targs, **tkwargs)
-        self.assertEqual(result, (arg, tuple(targs), tkwargs))
-        # raise ValueError(logger.mock_calls)
-        logger.assert_has_calls((
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Calling: \n'func'("
-                    "\n    'arg'={arg},"
-                    "\n    'positional'={args},"
-                    "\n    'named'={kwargs},\n)".format(
-                        arg=decorators.pretty_repr(
-                            arg,
-                            indent=8, no_indent_start=True),
-                        args=decorators.pretty_repr(
-                            tuple(targs),
-                            indent=8, no_indent_start=True),
-                        kwargs=decorators.pretty_repr(
-                            tkwargs,
-                            indent=8, no_indent_start=True)
-                    )
-            ),
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Done: 'func' with result:\n{}".format(
-                    decorators.pretty_repr(result))
-            ),
-        ))
-
-    def test_negative(self, logger):
-        @decorators.logwrap
-        def func():
-            raise ValueError('as expected')
-
-        with self.assertRaises(ValueError):
-            func()
-
-        logger.assert_has_calls((
-            mock.call.log(
-                level=logging.DEBUG,
-                msg="Calling: \n'func'()"
-            ),
-            mock.call.log(
-                level=logging.ERROR,
-                msg="Failed: \n'func'()",
-                exc_info=True
-            ),
-        ))
-
-    def test_negative_substitutions(self, logger):
-        new_logger = mock.Mock(spec=logging.Logger, name='logger')
-        log = mock.Mock(name='log')
-        new_logger.attach_mock(log, 'log')
-
-        @decorators.logwrap(
-            log=new_logger,
-            log_level=logging.INFO,
-            exc_level=logging.WARNING
-        )
-        def func():
-            raise ValueError('as expected')
-
-        with self.assertRaises(ValueError):
-            func()
-
-        self.assertEqual(len(logger.mock_calls), 0)
-        log.assert_has_calls((
-            mock.call(
-                level=logging.INFO,
-                msg="Calling: \n'func'()"
-            ),
-            mock.call(
-                level=logging.WARNING,
-                msg="Failed: \n'func'()",
-                exc_info=True
-            ),
-        ))
-
-
 class TestProcLock(unittest.TestCase):
 
     def patch(self, *args, **kwargs):

devops/tests/helpers/test_exec_result.py

@@ -1,215 +0,0 @@
-#    Copyright 2016 Mirantis, Inc.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-from __future__ import unicode_literals
-
-# pylint: disable=no-self-use
-
-import unittest
-
-import mock
-
-from devops import error
-from devops.helpers import exec_result
-from devops.helpers.proc_enums import ExitCodes
-from devops.helpers.subprocess_runner import Subprocess
-
-
-cmd = "ls -la | awk \'{print $1}\'"
-
-
-# noinspection PyTypeChecker
-class TestExecResult(unittest.TestCase):
-    @mock.patch('devops.helpers.exec_result.logger')
-    def test_create_minimal(self, logger):
-        """Test defaults"""
-        result = exec_result.ExecResult(cmd=cmd)
-        self.assertEqual(result.cmd, cmd)
-        self.assertEqual(result.cmd, result['cmd'])
-        self.assertEqual(result.stdout, [])
-        self.assertEqual(result.stdout, result['stdout'])
-        self.assertEqual(result.stderr, [])
-        self.assertEqual(result.stderr, result['stderr'])
-        self.assertEqual(result.stdout_bin, bytearray())
-        self.assertEqual(result.stderr_bin, bytearray())
-        self.assertEqual(result.stdout_str, '')
-        self.assertEqual(result.stdout_str, result['stdout_str'])
-        self.assertEqual(result.stderr_str, '')
-        self.assertEqual(result.stderr_str, result['stderr_str'])
-        self.assertEqual(result.stdout_brief, '')
-        self.assertEqual(result.stdout_brief, result['stdout_brief'])
-        self.assertEqual(result.stderr_brief, '')
-        self.assertEqual(result.stderr_brief, result['stderr_brief'])
-        self.assertEqual(result.exit_code, ExitCodes.EX_INVALID)
-        self.assertEqual(result.exit_code, result['exit_code'])
-        self.assertEqual(
-            repr(result),
-            '{cls}(cmd={cmd!r}, stdout={stdout}, stderr={stderr}, '
-            'exit_code={exit_code!s})'.format(
-                cls=exec_result.ExecResult.__name__,
-                cmd=cmd,
-                stdout=[],
-                stderr=[],
-                exit_code=ExitCodes.EX_INVALID
-            )
-        )
-        self.assertEqual(
-            str(result),
-            "{cls}(\n\tcmd={cmd!r},"
-            "\n\t stdout=\n'{stdout_brief}',"
-            "\n\tstderr=\n'{stderr_brief}', "
-            '\n\texit_code={exit_code!s}\n)'.format(
-                cls=exec_result.ExecResult.__name__,
-                cmd=cmd,
-                stdout_brief='',
-                stderr_brief='',
-                exit_code=ExitCodes.EX_INVALID
-            )
-        )
-
-        with self.assertRaises(IndexError):
-            # pylint: disable=pointless-statement
-            # noinspection PyStatementEffect
-            result['nonexistent']
-            # pylint: enable=pointless-statement
-
-        with self.assertRaises(error.DevopsError):
-            # pylint: disable=pointless-statement
-            # noinspection PyStatementEffect
-            result['stdout_json']
-            # pylint: enable=pointless-statement
-        logger.assert_has_calls((
-            mock.call.exception(
-                "{cmd} stdout is not valid json:\n"
-                "{stdout_str!r}\n".format(
-                    cmd=cmd,
-                    stdout_str='')),
-        ))
-        self.assertIsNone(result['stdout_yaml'])
-
-        self.assertEqual(
-            hash(result),
-            hash((exec_result.ExecResult, cmd, '', '', ExitCodes.EX_INVALID))
-        )
-
-    @mock.patch('devops.helpers.exec_result.logger', autospec=True)
-    def test_not_implemented(self, logger):
-        """Test assertion on non implemented deserializer"""
-        result = exec_result.ExecResult(cmd=cmd)
-        deserialize = getattr(result, '_ExecResult__deserialize')
-        with self.assertRaises(error.DevopsNotImplementedError):
-            deserialize('tst')
-        logger.assert_has_calls((
-            mock.call.error(
-                '{fmt} deserialize target is not implemented'.format(
-                    fmt='tst')),
-        ))
-
-    def test_setters(self):
-        result = exec_result.ExecResult(cmd=cmd)
-        self.assertEqual(result.exit_code, ExitCodes.EX_INVALID)
-        result.exit_code = 0
-        self.assertEqual(result.exit_code, 0)
-        self.assertEqual(result.exit_code, result['exit_code'])
-
-        tst_stdout = [
-            b'Test\n',
-            b'long\n',
-            b'stdout\n',
-            b'data\n',
-            b' \n',
-            b'5\n',
-            b'6\n',
-            b'7\n',
-            b'8\n',
-            b'end!\n'
-        ]
-
-        tst_stderr = [b'test\n'] * 10
-
-        result['stdout'] = tst_stdout
-        self.assertEqual(result.stdout, tst_stdout)
-        self.assertEqual(result.stdout, result['stdout'])
-
-        result['stderr'] = tst_stderr
-        self.assertEqual(result.stderr, tst_stderr)
-        self.assertEqual(result.stderr, result['stderr'])
-
-        with self.assertRaises(TypeError):
-            result.exit_code = 'code'
-
-        with self.assertRaises(error.DevopsError):
-            result['stdout_brief'] = 'test'
-
-        with self.assertRaises(IndexError):
-            result['test'] = True
-
-        with self.assertRaises(TypeError):
-            result.stdout = 'stdout'
-
-        self.assertEqual(result.stdout, tst_stdout)
-
-        with self.assertRaises(TypeError):
-            result.stderr = 'stderr'
-
-        self.assertEqual(result.stderr, tst_stderr)
-
-        self.assertEqual(result.stdout_bin, bytearray(b''.join(tst_stdout)))
-        self.assertEqual(result.stderr_bin, bytearray(b''.join(tst_stderr)))
-
-        stdout_br = tst_stdout[:3] + [b'...\n'] + tst_stdout[-3:]
-        stderr_br = tst_stderr[:3] + [b'...\n'] + tst_stderr[-3:]
-
-        stdout_brief = b''.join(stdout_br).strip().decode(encoding='utf-8')
-        stderr_brief = b''.join(stderr_br).strip().decode(encoding='utf-8')
-
-        self.assertEqual(result.stdout_brief, stdout_brief)
-        self.assertEqual(result.stderr_brief, stderr_brief)
-
-    def test_json(self):
-        result = exec_result.ExecResult('test', stdout=[b'{"test": true}'])
-        self.assertEqual(result.stdout_json, {'test': True})
-
-    @mock.patch('devops.helpers.exec_result.logger', autospec=True)
-    def test_deprecations(self, logger):
-        result = exec_result.ExecResult('test', stdout=[b'{"test": true}'])
-        for deprecated in ('stdout_json', 'stdout_yaml'):
-            result['{}'.format(deprecated)] = {'test': False}
-            logger.assert_has_calls((
-                mock.call.warning(
-                    '{key} is read-only and calculated automatically'.format(
-                        key='{}'.format(deprecated)
-                    )),
-            ))
-            self.assertEqual(result[deprecated], {'test': True})
-            logger.reset_mock()
-
-    @mock.patch('devops.helpers.exec_result.logger', autospec=True)
-    def test_wrong_result(self, logger):
-        """Test logging exception if stdout if not a correct json"""
-        cmd = "ls -la | awk \'{print $1\}\'"
-        result = Subprocess.execute(command=cmd)
-        with self.assertRaises(error.DevopsError):
-            # pylint: disable=pointless-statement
-            # noinspection PyStatementEffect
-            result.stdout_json
-            # pylint: enable=pointless-statement
-        logger.assert_has_calls((
-            mock.call.exception(
-                "{cmd} stdout is not valid json:\n"
-                "{stdout_str!r}\n".format(
-                    cmd=cmd,
-                    stdout_str='')),
-        ))
-        self.assertIsNone(result['stdout_yaml'])

devops/tests/helpers/test_helpers.py

@@ -19,6 +19,7 @@
 import socket
 import unittest
 
+import exec_helpers
 import mock
 
 # pylint: disable=redefined-builtin
@@ -27,9 +28,7 @@ from six.moves import xrange
 # pylint: enable=redefined-builtin
 
 from devops import error
-from devops.helpers import exec_result
 from devops.helpers import helpers
-from devops.helpers import ssh_client
 
 
 class TestHelpersHelpers(unittest.TestCase):
@@ -52,8 +51,8 @@ class TestHelpersHelpers(unittest.TestCase):
                 for port in xrange(32000, 32100)])
 
     @mock.patch(
-        'devops.helpers.subprocess_runner.Subprocess.execute',
+        'exec_helpers.Subprocess.execute',
-        return_value=exec_result.ExecResult(
+        return_value=exec_helpers.ExecResult(
             cmd="ping -c 1 -W '{timeout:d}' '{host:s}'".format(
                 host='127.0.0.1', timeout=1,
             ),
@@ -196,7 +195,7 @@ class TestHelpersHelpers(unittest.TestCase):
         helpers.wait_tcp(host, port, timeout)
         ping.assert_called_once_with(host=host, port=port)
 
-    @mock.patch('devops.helpers.ssh_client.SSHClient', autospec=True)
+    @mock.patch('exec_helpers.SSHClient', autospec=True)
     @mock.patch('devops.helpers.helpers.wait')
     def test_wait_ssh_cmd(self, wait, ssh):
         host = '127.0.0.1'
@@ -210,7 +209,7 @@ class TestHelpersHelpers(unittest.TestCase):
             host, port, check_cmd, username, password, timeout)
         ssh.assert_called_once_with(
             host=host, port=port,
-            auth=ssh_client.SSHAuth(username=username, password=password)
+            auth=exec_helpers.SSHAuth(username=username, password=password)
         )
         wait.assert_called_once()
         # Todo: cover ssh_client.execute

devops/tests/helpers/test_ntp.py

@@ -18,11 +18,11 @@
 
 import unittest
 
+import exec_helpers
 import mock
 
 from devops import error
 from devops.helpers import ntp
-from devops.helpers import ssh_client
 
 
 class NtpTestCase(unittest.TestCase):
@@ -34,18 +34,18 @@ class NtpTestCase(unittest.TestCase):
         return m
 
     def setUp(self):
-        self.remote_mock = mock.Mock(spec=ssh_client.SSHClient)
+        self.remote_mock = mock.Mock(spec=exec_helpers.SSHClient)
         self.remote_mock.__repr__ = mock.Mock(return_value='<SSHClient()>')
 
         self.wait_mock = self.patch('devops.helpers.helpers.wait')
 
     @staticmethod
     def make_exec_result(stdout, exit_code=0):
-        return {
+        return exec_helpers.ExecResult(
-            'exit_code': exit_code,
+            cmd='n/a',
-            'stderr': [],
+            exit_code=exit_code,
-            'stdout': stdout.splitlines(True),
+            stdout=stdout.splitlines(True)
-        }
+        )
 
 
 class TestNtpInitscript(NtpTestCase):
@@ -97,7 +97,7 @@ class TestNtpInitscript(NtpTestCase):
                 "find /etc/init.d/ -regex '/etc/init.d/ntp.?' -executable"),
             mock.call('ntpq -pn 127.0.0.1'),
         ))
-        assert peers == ['Line3\n', 'Line4\n']
+        assert peers == ('Line3\n', 'Line4\n')
 
     def test_date(self):
         self.remote_mock.execute.side_effect = (
@@ -214,7 +214,7 @@ class TestNtpPacemaker(NtpTestCase):
 
         self.remote_mock.execute.assert_called_once_with(
             'ip netns exec vrouter ntpq -pn 127.0.0.1')
-        assert peers == ['Line3\n', 'Line4\n']
+        assert peers == ('Line3\n', 'Line4\n')
 
 
 class TestNtpSystemd(NtpTestCase):

devops/tests/helpers/test_subprocess_runner.py

@@ -1,252 +0,0 @@
-# coding=utf-8
-
-#    Copyright 2016 Mirantis, Inc.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-from __future__ import unicode_literals
-
-import subprocess
-import unittest
-
-import mock
-
-from devops import error
-from devops.helpers import exec_result
-from devops.helpers import subprocess_runner
-
-command = 'ls ~\nline 2\nline 3\nline с кирилицей'
-command_log = u"Executing command:\n{!s}\n".format(command.rstrip())
-stdout_list = [b' \n', b'2\n', b'3\n', b' \n']
-stderr_list = [b' \n', b'0\n', b'1\n', b' \n']
-
-
-class FakeFileStream(object):
-    def __init__(self, *args):
-        self.__src = list(args)
-
-    def __iter__(self):
-        for _ in range(len(self.__src)):
-            yield self.__src.pop(0)
-
-    def fileno(self):
-        return hash(tuple(self.__src))
-
-
-# TODO(AStepanov): Cover negative scenarios (timeout)
-
-
-@mock.patch('devops.helpers.subprocess_runner.logger', autospec=True)
-@mock.patch('select.select', autospec=True)
-@mock.patch('fcntl.fcntl', autospec=True)
-@mock.patch('subprocess.Popen', autospec=True, name='subprocess.Popen')
-class TestSubprocessRunner(unittest.TestCase):
-    @staticmethod
-    def prepare_close(popen, stderr_val=None, ec=0):
-        stdout_lines = stdout_list
-        stderr_lines = stderr_list if stderr_val is None else []
-
-        stdout = FakeFileStream(*stdout_lines)
-        stderr = FakeFileStream(*stderr_lines)
-
-        popen_obj = mock.Mock()
-        popen_obj.attach_mock(stdout, 'stdout')
-        popen_obj.attach_mock(stderr, 'stderr')
-        popen_obj.configure_mock(returncode=ec)
-
-        popen.return_value = popen_obj
-
-        # noinspection PyTypeChecker
-        exp_result = exec_result.ExecResult(
-            cmd=command,
-            stderr=stderr_lines,
-            stdout=stdout_lines,
-            exit_code=ec
-            )
-
-        return popen_obj, exp_result
-
-    @staticmethod
-    def gen_cmd_result_log_message(result):
-        return (u"Command exit code '{code!s}':\n{cmd!s}\n"
-                .format(cmd=result.cmd.rstrip(), code=result.exit_code))
-
-    def test_call(self, popen, fcntl, select, logger):
-        popen_obj, exp_result = self.prepare_close(popen)
-        select.return_value = [popen_obj.stdout, popen_obj.stderr], [], []
-
-        runner = subprocess_runner.Subprocess()
-
-        # noinspection PyTypeChecker
-        result = runner.execute(command)
-        self.assertEqual(
-            result, exp_result
-
-        )
-        popen.assert_has_calls((
-            mock.call(
-                args=[command],
-                cwd=None,
-                env=None,
-                shell=True,
-                stderr=subprocess.PIPE,
-                stdin=subprocess.PIPE,
-                stdout=subprocess.PIPE,
-                universal_newlines=False),
-        ))
-        logger.assert_has_calls([
-            mock.call.debug(command_log),
-            ] + [
-                mock.call.debug(str(x.rstrip().decode('utf-8')))
-                for x in stdout_list
-            ] + [
-                mock.call.debug(str(x.rstrip().decode('utf-8')))
-                for x in stderr_list
-            ] + [
-            mock.call.debug(self.gen_cmd_result_log_message(result)),
-        ])
-        self.assertIn(
-            mock.call.poll(), popen_obj.mock_calls
-        )
-
-    def test_call_verbose(self, popen, fcntl, select, logger):
-        popen_obj, _ = self.prepare_close(popen)
-        select.return_value = [popen_obj.stdout, popen_obj.stderr], [], []
-
-        runner = subprocess_runner.Subprocess()
-
-        # noinspection PyTypeChecker
-        result = runner.execute(command, verbose=True)
-
-        logger.assert_has_calls([
-            mock.call.info(command_log),
-            ] + [
-                mock.call.info(str(x.rstrip().decode('utf-8')))
-                for x in stdout_list
-            ] + [
-                mock.call.error(str(x.rstrip().decode('utf-8')))
-                for x in stderr_list
-            ] + [
-            mock.call.info(self.gen_cmd_result_log_message(result)),
-        ])
-
-
-@mock.patch('devops.helpers.subprocess_runner.logger', autospec=True)
-class TestSubprocessRunnerHelpers(unittest.TestCase):
-    @mock.patch('devops.helpers.subprocess_runner.Subprocess.execute')
-    def test_check_call(self, execute, logger):
-        exit_code = 0
-        return_value = {
-            'stderr_str': '0\n1',
-            'stdout_str': '2\n3',
-            'stderr_brief': '0\n1',
-            'stdout_brief': '2\n3',
-            'exit_code': exit_code,
-            'stderr': [b' \n', b'0\n', b'1\n', b' \n'],
-            'stdout': [b' \n', b'2\n', b'3\n', b' \n']}
-        execute.return_value = return_value
-
-        verbose = False
-
-        runner = subprocess_runner.Subprocess()
-
-        # noinspection PyTypeChecker
-        result = runner.check_call(
-            command=command, verbose=verbose, timeout=None)
-        execute.assert_called_once_with(command, verbose, None)
-        self.assertEqual(result, return_value)
-
-        exit_code = 1
-        return_value['exit_code'] = exit_code
-        execute.reset_mock()
-        execute.return_value = return_value
-        with self.assertRaises(error.DevopsCalledProcessError):
-            # noinspection PyTypeChecker
-            runner.check_call(command=command, verbose=verbose, timeout=None)
-        execute.assert_called_once_with(command, verbose, None)
-
-    @mock.patch('devops.helpers.subprocess_runner.Subprocess.execute')
-    def test_check_call_expected(self, execute, logger):
-        exit_code = 0
-        return_value = {
-            'stderr_str': '0\n1',
-            'stdout_str': '2\n3',
-            'stderr_brief': '0\n1',
-            'stdout_brief': '2\n3',
-            'exit_code': exit_code,
-            'stderr': [b' \n', b'0\n', b'1\n', b' \n'],
-            'stdout': [b' \n', b'2\n', b'3\n', b' \n']}
-        execute.return_value = return_value
-
-        verbose = False
-
-        runner = subprocess_runner.Subprocess()
-
-        # noinspection PyTypeChecker
-        result = runner.check_call(
-            command=command, verbose=verbose, timeout=None, expected=[0, 75])
-        execute.assert_called_once_with(command, verbose, None)
-        self.assertEqual(result, return_value)
-
-        exit_code = 1
-        return_value['exit_code'] = exit_code
-        execute.reset_mock()
-        execute.return_value = return_value
-        with self.assertRaises(error.DevopsCalledProcessError):
-            # noinspection PyTypeChecker
-            runner.check_call(
-                command=command, verbose=verbose, timeout=None,
-                expected=[0, 75]
-            )
-        execute.assert_called_once_with(command, verbose, None)
-
-    @mock.patch('devops.helpers.subprocess_runner.Subprocess.check_call')
-    def test_check_stderr(self, check_call, logger):
-        return_value = {
-            'stderr_str': '',
-            'stdout_str': '2\n3',
-            'stderr_brief': '',
-            'stdout_brief': '2\n3',
-            'exit_code': 0,
-            'stderr': [],
-            'stdout': [b' \n', b'2\n', b'3\n', b' \n']}
-        check_call.return_value = return_value
-
-        verbose = False
-        raise_on_err = True
-
-        runner = subprocess_runner.Subprocess()
-
-        # noinspection PyTypeChecker
-        result = runner.check_stderr(
-            command=command, verbose=verbose, timeout=None,
-            raise_on_err=raise_on_err)
-        check_call.assert_called_once_with(
-            command, verbose, timeout=None,
-            error_info=None, raise_on_err=raise_on_err)
-        self.assertEqual(result, return_value)
-
-        return_value['stderr_str'] = '0\n1'
-        return_value['stderr_brief'] = '0\n1'
-        return_value['stderr'] = [b' \n', b'0\n', b'1\n', b' \n']
-
-        check_call.reset_mock()
-        check_call.return_value = return_value
-        with self.assertRaises(error.DevopsCalledProcessError):
-            # noinspection PyTypeChecker
-            runner.check_stderr(
-                command=command, verbose=verbose, timeout=None,
-                raise_on_err=raise_on_err)
-        check_call.assert_called_once_with(
-            command, verbose, timeout=None,
-            error_info=None, raise_on_err=raise_on_err)

devops/tests/models/node_ext/test_centos_master.py

@@ -82,8 +82,7 @@ class TestCentosMasterExt(LibvirtTestCase):
         self.wait_tcp_mock = self.patch(
             'devops.helpers.helpers.wait_tcp')
 
-    @mock.patch(
+    @mock.patch('exec_helpers.Subprocess', autospec=True)
-        'devops.helpers.subprocess_runner.Subprocess', autospec=True)
     @mock.patch('devops.driver.libvirt.libvirt_driver.uuid')
     @mock.patch('libvirt.virConnect.defineXML')
     @mock.patch.multiple(settings, CLOUD_IMAGE_DIR='/tmp/')

requirements.txt

@@ -0,0 +1,19 @@
+keystoneauth1>=2.1.0
+netaddr>=0.7.12,!=0.7.16
+paramiko>=1.16.0,!=2.0.1
+Django>=1.8,<1.9
+jsonfield
+PyYAML>=3.1.0
+libvirt-python>=3.5.0,<4.1.0
+tabulate
+six>=1.9.0
+python-dateutil>=2.4.2
+lxml
+enum34; python_version < "3.4"
+fasteners>=0.7.0
+virtualbmc
+
+tenacity
+logwrap
+exec-helpers>=3.1.4
+threaded

setup.py

@@ -12,11 +12,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import sys
-
 import setuptools
 
 
+with open("requirements.txt") as f:
+    required = f.read().splitlines()
+
+
 setuptools.setup(
     name='fuel-devops',
     version='3.0.5',
@@ -39,22 +41,7 @@ setuptools.setup(
     ],
     data_files=[('bin', ['bin/dos_functions.sh'])],
     # Use magic in install_requires due to risk of old setuptools
-    install_requires=[
+    install_requires=required,
-        'keystoneauth1>=2.1.0',
-        'netaddr>=0.7.12,!=0.7.16',
-        'paramiko>=1.16.0,!=2.0.1',
-        'Django>=1.8,<1.9',
-        'jsonfield',
-        'PyYAML>=3.1.0',
-        'libvirt-python>=3.5.0,<4.1.0',
-        'tabulate',
-        'six>=1.9.0',
-        'python-dateutil>=2.4.2',
-        'lxml',
-        'enum34' if sys.version_info.major == 2 else '',
-        'fasteners>=0.7.0',
-        'virtualbmc'
-    ],
     tests_require=[
         'pytest>=2.7.1',
         'pytest-django >= 2.8.0',

test-requirements.txt

@@ -1,4 +1,5 @@
+-r requirements.txt
 sphinx<=1.4.9
 mock>=1.2
 pytest>=2.7.1
-pytest-django >= 2.8.0
+pytest-django >= 2.8.0

tox.ini

@@ -5,13 +5,14 @@
 
 [tox]
 minversion = 2.0
-envlist = pep8, py{27,35}, pylint, pylint-py{27,35}, cover, docs
+envlist = pep8, py27, py3{5,6,7}, pylint, pylint-py{27,35}, cover, docs
 skipsdist = True
 skip_missing_interpreters = True
 
 
 [testenv]
 usedevelop = True
+recreate = True
 passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
 deps =
     -r{toxinidir}/test-requirements.txt