diff --git a/devops/client/environment.py b/devops/client/environment.py index 585d180f..2ce736a1 100644 --- a/devops/client/environment.py +++ b/devops/client/environment.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import exec_helpers import paramiko # pylint: disable=redefined-builtin # noinspection PyUnresolvedReferences @@ -22,7 +23,6 @@ from devops.client import nailgun from devops import error from devops.helpers import helpers from devops.helpers import ntp -from devops.helpers import ssh_client from devops.helpers import templates from devops import settings @@ -135,9 +135,9 @@ class DevopsEnvironment(object): host=admin_ip, port=admin_node.ssh_port, timeout=180, timeout_msg=("Admin node {ip} is not accessible by SSH." "".format(ip=admin_ip))) - return ssh_client.SSHClient( + return exec_helpers.SSHClient( admin_ip, - auth=ssh_client.SSHAuth(username=login, password=password)) + auth=exec_helpers.SSHAuth(username=login, password=password)) def get_private_keys(self): ssh_keys = [] @@ -165,9 +165,9 @@ class DevopsEnvironment(object): helpers.wait_tcp( host=ip, port=node.ssh_port, timeout=180, timeout_msg="Node {ip} is not accessible by SSH.".format(ip=ip)) - return ssh_client.SSHClient( + return exec_helpers.SSHClient( ip, - auth=ssh_client.SSHAuth( + auth=exec_helpers.SSHAuth( username=login, password=password, keys=self.get_private_keys())) @@ -191,9 +191,9 @@ class DevopsEnvironment(object): login=settings.SSH_SLAVE_CREDENTIALS['login'], password=settings.SSH_SLAVE_CREDENTIALS['password']): ip = self.find_node_ip(node_name) - return ssh_client.SSHClient( + return exec_helpers.SSHClient( ip, - auth=ssh_client.SSHAuth( + auth=exec_helpers.SSHAuth( username=login, password=password)) diff --git a/devops/driver/libvirt/libvirt_driver.py b/devops/driver/libvirt/libvirt_driver.py index dd812f19..752c4c6b 100644 --- a/devops/driver/libvirt/libvirt_driver.py +++ b/devops/driver/libvirt/libvirt_driver.py @@ -25,6 +25,7 @@ import xml.etree.ElementTree as ET from django.conf import settings from django.utils import functional +import exec_helpers import libvirt import netaddr import paramiko @@ -37,8 +38,6 @@ from devops.helpers import cloud_image_settings from devops.helpers import decorators from devops.helpers import helpers from devops.helpers import scancodes -from devops.helpers import ssh_client -from devops.helpers import subprocess_runner from devops import logger from devops.models import base from devops.models import driver @@ -355,16 +354,16 @@ class LibvirtDriver(driver.Driver): logger.debug("Initializing SSHClient for username:'{0}', host:" "'{1}', port:'{2}'".format(username, host, port)) keys = [key] - return ssh_client.SSHClient( + return exec_helpers.SSHClient( host=host, port=port, - auth=ssh_client.SSHAuth( + auth=exec_helpers.SSHAuth( username=username, keys=keys)) else: # Using SubprocessClient to execute shell commands on local host logger.debug("Initializing subprocess_runner for local host") - return subprocess_runner.Subprocess + return exec_helpers.Subprocess() class LibvirtL2NetworkDevice(network.L2NetworkDevice): @@ -696,7 +695,7 @@ class LibvirtL2NetworkDevice(network.L2NetworkDevice): # of this try/except workaround try: self.driver.shell.check_call(cmd) - except error.DevopsCalledProcessError: + except exec_helpers.CalledProcessError: pass @decorators.retry(libvirt.libvirtError) diff --git a/devops/driver/libvirt/libvirt_xml_builder.py b/devops/driver/libvirt/libvirt_xml_builder.py index 9bb81fca..ef9e5459 100644 --- a/devops/driver/libvirt/libvirt_xml_builder.py +++ b/devops/driver/libvirt/libvirt_xml_builder.py @@ -15,10 +15,14 @@ from __future__ import unicode_literals import hashlib +import logwrap import six -from devops.helpers.decorators import logwrap from devops.helpers import xmlgenerator +from devops import logger + + +log_call = logwrap.logwrap(log=logger) class LibvirtXMLBuilder(object): @@ -36,7 +40,7 @@ class LibvirtXMLBuilder(object): return name @classmethod - @logwrap + @log_call def build_network_xml(cls, network_name, bridge_name, addresses=None, forward=None, ip_network_address=None, ip_network_prefixlen=None, stp=True, @@ -92,7 +96,7 @@ class LibvirtXMLBuilder(object): return str(network_xml) @classmethod - @logwrap + @log_call def build_volume_xml(cls, name, capacity, vol_format, backing_store_path, backing_store_format): """Generate volume XML @@ -112,7 +116,7 @@ class LibvirtXMLBuilder(object): return str(volume_xml) @classmethod - @logwrap + @log_call def build_snapshot_xml(cls, name=None, description=None, external=False, disk_only=False, memory_file='', domain_isactive=False, local_disk_devices=None): @@ -198,7 +202,7 @@ class LibvirtXMLBuilder(object): device_xml.filterref(filter=interface_filter) @classmethod - @logwrap + @log_call def build_network_filter(cls, name, uuid=None, rule=None): """Generate nwfilter XML for network @@ -216,7 +220,7 @@ class LibvirtXMLBuilder(object): return str(filter_xml) @classmethod - @logwrap + @log_call def build_interface_filter(cls, name, filterref, uuid=None, rule=None): """Generate nwfilter XML for interface @@ -236,7 +240,7 @@ class LibvirtXMLBuilder(object): return str(filter_xml) @classmethod - @logwrap + @log_call def build_node_xml(cls, name, hypervisor, use_host_cpu, vcpu, memory, use_hugepages, hpet, os_type, architecture, boot, reboot_timeout, bootmenu_timeout, emulator, @@ -333,7 +337,7 @@ class LibvirtXMLBuilder(object): return str(node_xml) @classmethod - @logwrap + @log_call def build_iface_xml(cls, name, ip=None, prefix=None, vlanid=None): """Generate interface bridge XML diff --git a/devops/error.py b/devops/error.py index 0e117bf1..992a4a78 100644 --- a/devops/error.py +++ b/devops/error.py @@ -17,7 +17,7 @@ from __future__ import unicode_literals import inspect import warnings -import six +import exec_helpers class DevopsException(Exception): @@ -35,36 +35,7 @@ class AuthenticationError(DevopsError): pass -class DevopsCalledProcessError(DevopsError): - @staticmethod - def _makestr(data): - if isinstance(data, six.binary_type): - return data.decode('utf-8', errors='backslashreplace') - elif isinstance(data, six.text_type): - return data - else: - return repr(data) - - def __init__( - self, command, returncode, expected=0, stdout=None, stderr=None): - self.returncode = returncode - self.expected = expected - self.cmd = command - self.stdout = stdout - self.stderr = stderr - message = ( - "Command '{cmd}' returned exit code {code} while " - "expected {expected}".format( - cmd=self._makestr(self.cmd), - code=self.returncode, - expected=self.expected - )) - if self.stdout: - message += "\n\tSTDOUT:\n{}".format(self._makestr(self.stdout)) - if self.stderr: - message += "\n\tSTDERR:\n{}".format(self._makestr(self.stderr)) - super(DevopsCalledProcessError, self).__init__(message) - +class DevopsCalledProcessError(DevopsError, exec_helpers.CalledProcessError): @property def output(self): warnings.warn( diff --git a/devops/helpers/cloud_image_settings.py b/devops/helpers/cloud_image_settings.py index cf0e87b0..a959c85a 100644 --- a/devops/helpers/cloud_image_settings.py +++ b/devops/helpers/cloud_image_settings.py @@ -14,9 +14,9 @@ import os -from devops.helpers.helpers import format_data -from devops.helpers import subprocess_runner +import exec_helpers +from devops.helpers.helpers import format_data from devops import logger @@ -92,4 +92,4 @@ def generate_cloud_image_settings(cloud_image_settings_path, meta_data_path, user_data_path, meta_data_path) - subprocess_runner.Subprocess.check_call(cmd) + exec_helpers.Subprocess().check_call(cmd) diff --git a/devops/helpers/decorators.py b/devops/helpers/decorators.py index c4ef22a9..d867cde5 100644 --- a/devops/helpers/decorators.py +++ b/devops/helpers/decorators.py @@ -19,11 +19,13 @@ import functools import inspect import logging import sys -import threading import time +import warnings import fasteners +import logwrap as ext_logwrap import six +import threaded as ext_threaded from devops import error from devops import logger @@ -31,43 +33,12 @@ from devops import settings def threaded(name=None, started=False, daemon=False): - """Make function or method threaded with passing arguments - - If decorator added not as function, name is generated from function name. - - :type name: str - :type started: bool - :type daemon: bool - """ - - def real_decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - """Thread generator for function - - :rtype: Thread - """ - if name is None: - func_name = 'Threaded {}'.format(func.__name__) - else: - func_name = name - thread = threading.Thread( - target=func, - name=func_name, - args=args, - kwargs=kwargs) - if daemon: - thread.daemon = True - if started: - thread.start() - return thread - return wrapper - - if name is not None and callable(name): - func, name = name, None - return real_decorator(func) - - return real_decorator + warnings.warn( + 'helpers.decorators.threaded is deprecated' + ' in favor of external threaded', + DeprecationWarning + ) + return ext_threaded.threaded(name=name, started=started, daemon=daemon) def retry(exception, count=10, delay=1): @@ -177,148 +148,32 @@ def _getcallargs(func, *positional, **named): # pylint:enable=no-member -def _simple(item): - """Check for nested iterations: True, if not""" - return not isinstance(item, (list, set, tuple, dict)) - - -_formatters = { - 'simple': "{spc:<{indent}}{val!r}".format, - 'text': "{spc:<{indent}}{prefix}'''{string}'''".format, - 'dict': "\n{spc:<{indent}}{key!r:{size}}: {val},".format, - } - - def pretty_repr(src, indent=0, no_indent_start=False, max_indent=20): - """Make human readable repr of object - - :param src: object to process - :type src: object - :param indent: start indentation, all next levels is +4 - :type indent: int - :param no_indent_start: do not indent open bracket and simple parameters - :type no_indent_start: bool - :param max_indent: maximal indent before classic repr() call - :type max_indent: int - :return: formatted string - """ - if _simple(src) or indent >= max_indent: - indent = 0 if no_indent_start else indent - if isinstance(src, (six.binary_type, six.text_type)): - if isinstance(src, six.binary_type): - string = src.decode( - encoding='utf-8', - errors='backslashreplace' - ) - prefix = 'b' - else: - string = src - prefix = 'u' - return _formatters['text']( - spc='', - indent=indent, - prefix=prefix, - string=string - ) - return _formatters['simple']( - spc='', - indent=indent, - val=src - ) - if isinstance(src, dict): - prefix, suffix = '{', '}' - result = '' - max_len = len(max([repr(key) for key in src])) if src else 0 - for key, val in src.items(): - result += _formatters['dict']( - spc='', - indent=indent + 4, - size=max_len, - key=key, - val=pretty_repr(val, indent + 8, no_indent_start=True) - ) - return ( - '\n{start:>{indent}}'.format( - start=prefix, - indent=indent + 1 - ) + - result + - '\n{end:>{indent}}'.format(end=suffix, indent=indent + 1) - ) - if isinstance(src, list): - prefix, suffix = '[', ']' - elif isinstance(src, tuple): - prefix, suffix = '(', ')' - else: - prefix, suffix = '{', '}' - result = '' - for elem in src: - if _simple(elem): - result += '\n' - result += pretty_repr(elem, indent + 4) + ',' - return ( - '\n{start:>{indent}}'.format( - start=prefix, - indent=indent + 1) + - result + - '\n{end:>{indent}}'.format(end=suffix, indent=indent + 1) + warnings.warn( + 'helpers.decorators.pretty_repr is deprecated' + ' in favor of external logwrap', + DeprecationWarning + ) + return ext_logwrap.pretty_repr( + src=src, + indent=indent, + no_indent_start=no_indent_start, + max_indent=max_indent ) def logwrap(log=logger, log_level=logging.DEBUG, exc_level=logging.ERROR): - """Log function calls - - :type log: logging.Logger - :type log_level: int - :type exc_level: int - :rtype: callable - """ - def real_decorator(func): - @functools.wraps(func) - def wrapped(*args, **kwargs): - call_args = _getcallargs(func, *args, **kwargs) - args_repr = "" - if len(call_args) > 0: - args_repr = "\n " + "\n ".join(( - "{key!r}={val},".format( - key=key, - val=pretty_repr(val, indent=8, no_indent_start=True) - ) - for key, val in call_args.items()) - ) + '\n' - log.log( - level=log_level, - msg="Calling: \n{name!r}({arguments})".format( - name=func.__name__, - arguments=args_repr - ) - ) - try: - result = func(*args, **kwargs) - log.log( - level=log_level, - msg="Done: {name!r} with result:\n{result}".format( - name=func.__name__, - result=pretty_repr(result)) - ) - except BaseException: - log.log( - level=exc_level, - msg="Failed: \n{name!r}({arguments})".format( - name=func.__name__, - arguments=args_repr, - ), - exc_info=True - ) - raise - return result - return wrapped - - if not isinstance(log, logging.Logger): - func, log = log, logger - return real_decorator(func) - - return real_decorator + warnings.warn( + 'helpers.decorators.logwrap is deprecated' + ' in favor of external logwrap', + DeprecationWarning + ) + return ext_logwrap.logwrap( + func=log if callable(log) else None, + log=log if isinstance(log, logging.Logger) else logger, + log_level=log_level, + exc_level=exc_level + ) def proc_lock(path=settings.DEVOPS_LOCK_FILE, timeout=300): diff --git a/devops/helpers/exec_result.py b/devops/helpers/exec_result.py index 5bce6f00..276672a9 100644 --- a/devops/helpers/exec_result.py +++ b/devops/helpers/exec_result.py @@ -12,369 +12,17 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import absolute_import from __future__ import unicode_literals -import json -import threading +import warnings -import yaml +from exec_helpers import ExecResult -from devops import error -from devops.helpers import proc_enums -from devops import logger +__all__ = ('ExecResult',) - -deprecated_aliases = { - 'stdout_str', - 'stderr_str', - 'stdout_json', - 'stdout_yaml' -} - - -class ExecResult(object): - __slots__ = [ - '__cmd', '__stdout', '__stderr', '__exit_code', - '__stdout_str', '__stderr_str', '__stdout_brief', '__stderr_brief', - '__stdout_json', '__stdout_yaml', - '__lock' - ] - - def __init__(self, cmd, stdout=None, stderr=None, - exit_code=proc_enums.ExitCodes.EX_INVALID): - """Command execution result read from fifo - - :type cmd: str - :type stdout: list - :type stderr: list - :type exit_code: ExitCodes - """ - self.__lock = threading.RLock() - - self.__cmd = cmd - self.__stdout = stdout if stdout is not None else [] - self.__stderr = stderr if stderr is not None else [] - - self.__exit_code = None - self.exit_code = exit_code - - # By default is none: - self.__stdout_str = None - self.__stderr_str = None - self.__stdout_brief = None - self.__stderr_brief = None - - self.__stdout_json = None - self.__stdout_yaml = None - - @property - def lock(self): - """Lock object for thread-safe operation - - :rtype: RLock - """ - return self.__lock - - @staticmethod - def _get_bytearray_from_array(src): - """Get bytearray from array of bytes blocks - - :type src: list(bytes) - :rtype: bytearray - """ - return bytearray(b''.join(src)) - - @staticmethod - def _get_str_from_bin(src): - """Join data in list to the string, with python 2&3 compatibility. - - :type src: bytearray - :rtype: str - """ - return src.strip().decode( - encoding='utf-8', - errors='backslashreplace' - ) - - @classmethod - def _get_brief(cls, data): - """Get brief output: 7 lines maximum (3 first + ... + 3 last) - - :type data: list(bytes) - :rtype: str - """ - src = data if len(data) <= 7 else data[:3] + [b'...\n'] + data[-3:] - return cls._get_str_from_bin( - cls._get_bytearray_from_array(src) - ) - - @property - def cmd(self): - """Executed command - - :rtype: str - """ - return self.__cmd - - @property - def stdout(self): - """Stdout output as list of binaries - - :rtype: list(bytes) - """ - return self.__stdout - - @stdout.setter - def stdout(self, new_val): - """Stdout output as list of binaries - - :type new_val: list(bytes) - :raises: TypeError - """ - if not isinstance(new_val, (list, type(None))): - raise TypeError('stdout should be list only!') - with self.lock: - self.__stdout_str = None - self.__stdout_brief = None - self.__stdout_json = None - self.__stdout_yaml = None - self.__stdout = new_val - - @property - def stderr(self): - """Stderr output as list of binaries - - :rtype: list(bytes) - """ - return self.__stderr - - @stderr.setter - def stderr(self, new_val): - """Stderr output as list of binaries - - :type new_val: list(bytes) - :raises: TypeError - """ - if not isinstance(new_val, (list, None)): - raise TypeError('stderr should be list only!') - with self.lock: - self.__stderr_str = None - self.__stderr_brief = None - self.__stderr = new_val - - @property - def stdout_bin(self): - """Stdout in binary format - - Sometimes logging is used to log binary objects too (example: Session), - and for debug purposes we can use this as data source. - :rtype: bytearray - """ - with self.lock: - return self._get_bytearray_from_array(self.stdout) - - @property - def stderr_bin(self): - """Stderr in binary format - - :rtype: bytearray - """ - with self.lock: - return self._get_bytearray_from_array(self.stderr) - - @property - def stdout_str(self): - """Stdout output as string - - :rtype: str - """ - with self.lock: - if self.__stdout_str is None: - self.__stdout_str = self._get_str_from_bin(self.stdout_bin) - return self.__stdout_str - - @property - def stderr_str(self): - """Stderr output as string - - :rtype: str - """ - with self.lock: - if self.__stderr_str is None: - self.__stderr_str = self._get_str_from_bin(self.stderr_bin) - return self.__stderr_str - - @property - def stdout_brief(self): - """Brief stdout output (mostly for exceptions) - - :rtype: str - """ - with self.lock: - if self.__stdout_brief is None: - self.__stdout_brief = self._get_brief(self.stdout) - return self.__stdout_brief - - @property - def stderr_brief(self): - """Brief stderr output (mostly for exceptions) - - :rtype: str - """ - with self.lock: - if self.__stderr_brief is None: - self.__stderr_brief = self._get_brief(self.stderr) - return self.__stderr_brief - - @property - def exit_code(self): - """Return(exit) code of command - - :rtype: int - """ - return self.__exit_code - - @exit_code.setter - def exit_code(self, new_val): - """Return(exit) code of command - - :type new_val: int - """ - if not isinstance(new_val, (int, proc_enums.ExitCodes)): - raise TypeError('Exit code is strictly int') - with self.lock: - if isinstance(new_val, int) and \ - new_val in proc_enums.ExitCodes.__members__.values(): - new_val = proc_enums.ExitCodes(new_val) - self.__exit_code = new_val - - def __deserialize(self, fmt): - """Deserialize stdout as data format - - :type fmt: str - :rtype: object - :raises: DevopsError - """ - try: - if fmt == 'json': - return json.loads(self.stdout_str, encoding='utf-8') - elif fmt == 'yaml': - return yaml.safe_load(self.stdout_str) - except BaseException: - tmpl = ( - " stdout is not valid {fmt}:\n" - '{{stdout!r}}\n'.format( - fmt=fmt)) - logger.exception(self.cmd + tmpl.format(stdout=self.stdout_str)) - raise error.DevopsError( - self.cmd + tmpl.format(stdout=self.stdout_brief)) - msg = '{fmt} deserialize target is not implemented'.format(fmt=fmt) - logger.error(msg) - raise error.DevopsNotImplementedError(msg) - - @property - def stdout_json(self): - """JSON from stdout - - :rtype: object - """ - with self.lock: - if self.__stdout_json is None: - # noinspection PyTypeChecker - self.__stdout_json = self.__deserialize(fmt='json') - return self.__stdout_json - - @property - def stdout_yaml(self): - """YAML from stdout - - :rtype: Union(list, dict, None) - """ - with self.lock: - if self.__stdout_yaml is None: - # noinspection PyTypeChecker - self.__stdout_yaml = self.__deserialize(fmt='yaml') - return self.__stdout_yaml - - def __dir__(self): - return [ - 'cmd', 'stdout', 'stderr', 'exit_code', - 'stdout_bin', 'stderr_bin', - 'stdout_str', 'stderr_str', 'stdout_brief', 'stderr_brief', - 'stdout_json', 'stdout_yaml', - 'lock' - ] - - def __getitem__(self, item): - if item in dir(self): - return getattr(self, item) - raise IndexError( - '"{item}" not found in {dir}'.format( - item=item, dir=dir(self) - ) - ) - - def __setitem__(self, key, value): - rw = ['stdout', 'stderr', 'exit_code'] - if key in rw: - setattr(self, key, value) - return - if key in deprecated_aliases: - logger.warning( - '{key} is read-only and calculated automatically'.format( - key=key - ) - ) - return - if key in dir(self): - raise error.DevopsError( - '{key} is read-only!'.format(key=key) - ) - raise IndexError( - '{key} not found in {dir}'.format( - key=key, dir=rw - ) - ) - - def __repr__(self): - return ( - '{cls}(cmd={cmd!r}, stdout={stdout}, stderr={stderr}, ' - 'exit_code={exit_code!s})'.format( - cls=self.__class__.__name__, - cmd=self.cmd, - stdout=self.stdout, - stderr=self.stderr, - exit_code=self.exit_code - )) - - def __str__(self): - return ( - "{cls}(\n\tcmd={cmd!r}," - "\n\t stdout=\n'{stdout_brief}'," - "\n\tstderr=\n'{stderr_brief}', " - '\n\texit_code={exit_code!s}\n)'.format( - cls=self.__class__.__name__, - cmd=self.cmd, - stdout_brief=self.stdout_brief, - stderr_brief=self.stderr_brief, - exit_code=self.exit_code - ) - ) - - def __eq__(self, other): - return all( - ( - getattr(self, val) == getattr(other, val) - for val in ['cmd', 'stdout', 'stderr', 'exit_code'] - ) - ) - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash( - ( - self.__class__, self.cmd, self.stdout_str, self.stderr_str, - self.exit_code - )) +warnings.warn( + 'devops.helpers.exec_result.ExecResult is deprecated' + ' in favor of external exec_helpers', + DeprecationWarning +) diff --git a/devops/helpers/helpers.py b/devops/helpers/helpers.py index b14bf942..096565ab 100644 --- a/devops/helpers/helpers.py +++ b/devops/helpers/helpers.py @@ -25,6 +25,7 @@ import warnings import xml.etree.ElementTree as ET from dateutil import tz +import exec_helpers import six # pylint: disable=import-error # noinspection PyUnresolvedReferences @@ -34,8 +35,6 @@ from six.moves import xmlrpc_client # pylint: enable=import-error from devops import error -from devops.helpers import ssh_client -from devops.helpers import subprocess_runner from devops import logger from devops import settings @@ -53,7 +52,7 @@ def icmp_ping(host, timeout=1): returns True if host is pingable False - otherwise. """ - result = subprocess_runner.Subprocess.execute( + result = exec_helpers.Subprocess().execute( "ping -c 1 -W '{timeout:d}' '{host:s}'".format( host=host, timeout=timeout)) return result.exit_code == 0 @@ -238,10 +237,10 @@ def wait_ssh_cmd( username=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password'], timeout=0): - ssh = ssh_client.SSHClient(host=host, port=port, - auth=ssh_client.SSHAuth( - username=username, - password=password)) + ssh = exec_helpers.SSHClient(host=host, port=port, + auth=exec_helpers.SSHAuth( + username=username, + password=password)) wait(lambda: not ssh.execute(check_cmd)['exit_code'], timeout=timeout) diff --git a/devops/helpers/log_templates.py b/devops/helpers/log_templates.py deleted file mode 100644 index b4f8147c..00000000 --- a/devops/helpers/log_templates.py +++ /dev/null @@ -1,26 +0,0 @@ -# coding=utf-8 - -# Copyright 2017 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -CMD_EXEC = u"Executing command:\n{cmd!s}\n" -CMD_RESULT = (u"Command exit code '{code!s}':\n{cmd!s}\n") -CMD_UNEXPECTED_EXIT_CODE = (u"{append}Command '{cmd!s}' returned " - u"exit code '{code!s}' while " - u"expected '{expected!s}'\n") -CMD_UNEXPECTED_STDERR = (u"{append}Command '{cmd!s}' STDERR while " - u"not expected\n" - u"\texit code: '{code!s}'") -CMD_WAIT_ERROR = (u"Wait for '{cmd!s}' during {timeout!s}s: " - u"no return code!") diff --git a/devops/helpers/proc_enums.py b/devops/helpers/proc_enums.py index 73518fcc..5b358a16 100644 --- a/devops/helpers/proc_enums.py +++ b/devops/helpers/proc_enums.py @@ -12,113 +12,17 @@ # License for the specific language governing permissions and limitations # under the License. -import enum +from __future__ import absolute_import +from __future__ import unicode_literals +import warnings -@enum.unique -class SigNum(enum.IntEnum): - SIGHUP = 1 # Hangup (POSIX). - SIGINT = 2 # Interrupt (ANSI). - SIGQUIT = 3 # Quit (POSIX). - SIGILL = 4 # Illegal instruction (ANSI). - SIGTRAP = 5 # Trace trap (POSIX). - SIGABRT = 6 # Abort (ANSI). - SIGBUS = 7 # BUS error (4.2 BSD). - SIGFPE = 8 # Floating-point exception (ANSI). - SIGKILL = 9 # Kill, unblockable (POSIX). - SIGUSR1 = 10 # User-defined signal 1 (POSIX). - SIGSEGV = 11 # Segmentation violation (ANSI). - SIGUSR2 = 12 # User-defined signal 2 (POSIX). - SIGPIPE = 13 # Broken pipe (POSIX). - SIGALRM = 14 # Alarm clock (POSIX). - SIGTERM = 15 # Termination (ANSI). - SIGSTKFLT = 16 # Stack fault. - SIGCHLD = 17 # Child status has changed (POSIX). - SIGCONT = 18 # Continue (POSIX). - SIGSTOP = 19 # Stop, unblockable (POSIX). - SIGTSTP = 20 # Keyboard stop (POSIX). - SIGTTIN = 21 # Background read from tty (POSIX). - SIGTTOU = 22 # Background write to tty (POSIX). - SIGURG = 23 # Urgent condition on socket (4.2 BSD). - SIGXCPU = 24 # CPU limit exceeded (4.2 BSD). - SIGXFSZ = 25 # File size limit exceeded (4.2 BSD). - SIGVTALRM = 26 # Virtual alarm clock (4.2 BSD). - SIGPROF = 27 # Profiling alarm clock (4.2 BSD). - SIGWINCH = 28 # Window size change (4.3 BSD, Sun). - SIGPOLL = 29 # Pollable event occurred (System V) - SIGPWR = 30 # Power failure restart (System V). - SIGSYS = 31 # Bad system call. +from exec_helpers import ExitCodes - def __str__(self): - return "{name}<{value:d}(0x{value:02X})>".format( - name=self.name, - value=self.value - ) +__all__ = ('ExitCodes',) - -@enum.unique -class ExitCodes(enum.IntEnum): - EX_OK = 0 # successful termination - - EX_INVALID = 0xDEADBEEF # uint32 debug value. Impossible for POSIX - - EX_ERROR = 1 # general failure - EX_BUILTIN = 2 # Misuse of shell builtins (according to Bash) - - EX_USAGE = 64 # command line usage error - EX_DATAERR = 65 # data format error - EX_NOINPUT = 66 # cannot open input - EX_NOUSER = 67 # addressee unknown - EX_NOHOST = 68 # host name unknown - EX_UNAVAILABLE = 69 # service unavailable - EX_SOFTWARE = 70 # internal software error - EX_OSERR = 71 # system error (e.g., can't fork) - EX_OSFILE = 72 # critical OS file missing - EX_CANTCREAT = 73 # can't create (user) output file - EX_IOERR = 74 # input/output error - EX_TEMPFAIL = 75 # temp failure; user is invited to retry - EX_PROTOCOL = 76 # remote error in protocol - EX_NOPERM = 77 # permission denied - EX_CONFIG = 78 # configuration error - - EX_NOEXEC = 126 # If a command is found but is not executable - EX_NOCMD = 127 # If a command is not found - - # Signal exits: - EX_SIGHUP = 128 + SigNum.SIGHUP - EX_SIGINT = 128 + SigNum.SIGINT - EX_SIGQUIT = 128 + SigNum.SIGQUIT - EX_SIGILL = 128 + SigNum.SIGILL - EX_SIGTRAP = 128 + SigNum.SIGTRAP - EX_SIGABRT = 128 + SigNum.SIGABRT - EX_SIGBUS = 128 + SigNum.SIGBUS - EX_SIGFPE = 128 + SigNum.SIGFPE - EX_SIGKILL = 128 + SigNum.SIGKILL - EX_SIGUSR1 = 128 + SigNum.SIGUSR1 - EX_SIGSEGV = 128 + SigNum.SIGSEGV - EX_SIGUSR2 = 128 + SigNum.SIGUSR2 - EX_SIGPIPE = 128 + SigNum.SIGPIPE - EX_SIGALRM = 128 + SigNum.SIGALRM - EX_SIGTERM = 128 + SigNum.SIGTERM - EX_SIGSTKFLT = 128 + SigNum.SIGSTKFLT - EX_SIGCHLD = 128 + SigNum.SIGCHLD - EX_SIGCONT = 128 + SigNum.SIGCONT - EX_SIGSTOP = 128 + SigNum.SIGSTOP - EX_SIGTSTP = 128 + SigNum.SIGTSTP - EX_SIGTTIN = 128 + SigNum.SIGTTIN - EX_SIGTTOU = 128 + SigNum.SIGTTOU - EX_SIGURG = 128 + SigNum.SIGURG - EX_SIGXCPU = 128 + SigNum.SIGXCPU - EX_SIGXFSZ = 128 + SigNum.SIGXFSZ - EX_SIGVTALRM = 128 + SigNum.SIGVTALRM - EX_SIGPROF = 128 + SigNum.SIGPROF - EX_SIGWINCH = 128 + SigNum.SIGWINCH - EX_SIGPOLL = 128 + SigNum.SIGPOLL - EX_SIGPWR = 128 + SigNum.SIGPWR - EX_SIGSYS = 128 + SigNum.SIGSYS - - def __str__(self): - return "{name}<{value:d}(0x{value:02X})>".format( - name=self.name, - value=self.value - ) +warnings.warn( + 'devops.helpers.proc_enums.ExitCodes is deprecated' + ' in favor of external exec_helpers', + DeprecationWarning +) diff --git a/devops/helpers/retry.py b/devops/helpers/retry.py deleted file mode 100644 index 9c1169be..00000000 --- a/devops/helpers/retry.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2013 - 2016 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import warnings - -from devops.helpers.decorators import retry -from devops import logger - - -msg = 'devops.helpers.retry.retry was moved to devops.helpers.decorators.retry' -logger.critical(msg) -warnings.warn(msg, DeprecationWarning) - -__all__ = ['retry'] diff --git a/devops/helpers/ssh_client.py b/devops/helpers/ssh_client.py index 40f49601..0b68d389 100644 --- a/devops/helpers/ssh_client.py +++ b/devops/helpers/ssh_client.py @@ -12,1061 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import absolute_import from __future__ import unicode_literals -import base64 -import os -import posixpath -import stat -import sys -import threading -import time import warnings -import paramiko -import six +from exec_helpers import SSHAuth +from exec_helpers import SSHClient -from devops import error -from devops.helpers import decorators -from devops.helpers import exec_result -from devops.helpers import log_templates -from devops.helpers import proc_enums -from devops import logger +__all__ = ('SSHAuth', 'SSHClient') -class SSHAuth(object): - __slots__ = ['__username', '__password', '__key', '__keys'] - - def __init__( - self, - username=None, password=None, key=None, keys=None): - """SSH authorisation object - - Used to authorize SSHClient. - Single SSHAuth object is associated with single host:port. - Password and key is private, other data is read-only. - - :type username: str - :type password: str - :type key: paramiko.RSAKey - :type keys: list - """ - self.__username = username - self.__password = password - self.__key = key - self.__keys = [None] - if key is not None: - # noinspection PyTypeChecker - self.__keys.append(key) - if keys is not None: - for key in keys: - if key not in self.__keys: - self.__keys.append(key) - - @property - def username(self): - """Username for auth - - :rtype: str - """ - return self.__username - - @staticmethod - def __get_public_key(key): - """Internal method for get public key from private - - :type key: paramiko.RSAKey - """ - if key is None: - return None - return '{0} {1}'.format(key.get_name(), key.get_base64()) - - @property - def public_key(self): - """public key for stored private key if presents else None - - :rtype: str - """ - return self.__get_public_key(self.__key) - - def enter_password(self, tgt): - """Enter password to STDIN - - Note: required for 'sudo' call - - :type tgt: file - :rtype: str - """ - # noinspection PyTypeChecker - return tgt.write('{}\n'.format(self.__password)) - - def connect(self, client, hostname=None, port=22, log=True): - """Connect SSH client object using credentials - - :type client: - paramiko.client.SSHClient - paramiko.transport.Transport - :type log: bool - :raises paramiko.AuthenticationException - """ - kwargs = { - 'username': self.username, - 'password': self.__password} - if hostname is not None: - kwargs['hostname'] = hostname - kwargs['port'] = port - - keys = [self.__key] - keys.extend([k for k in self.__keys if k != self.__key]) - - for key in keys: - kwargs['pkey'] = key - try: - client.connect(**kwargs) - if self.__key != key: - self.__key = key - logger.debug( - 'Main key has been updated, public key is: \n' - '{}'.format(self.public_key)) - return - except paramiko.PasswordRequiredException: - if self.__password is None: - logger.exception('No password has been set!') - raise - else: - logger.critical( - 'Unexpected PasswordRequiredException, ' - 'when password is set!') - raise - except (paramiko.AuthenticationException, - paramiko.BadHostKeyException): - continue - msg = 'Connection using stored authentication info failed!' - if log: - logger.exception( - 'Connection using stored authentication info failed!') - raise paramiko.AuthenticationException(msg) - - def __hash__(self): - return hash(( - self.__class__, - self.username, - self.__password, - tuple(self.__keys) - )) - - def __eq__(self, other): - return hash(self) == hash(other) - - def __ne__(self, other): - return not self.__eq__(other) - - def __deepcopy__(self, memo): - return self.__class__( - username=self.username, - password=self.__password, - key=self.__key, - keys=self.__keys.copy() - ) - - def copy(self): - return self.__class__( - username=self.username, - password=self.__password, - key=self.__key, - keys=self.__keys - ) - - def __repr__(self): - _key = ( - None if self.__key is None else - ''.format(self.public_key) - ) - _keys = [] - for k in self.__keys: - if k == self.__key: - continue - # noinspection PyTypeChecker - _keys.append( - ''.format( - self.__get_public_key(key=k)) if k is not None else None) - - return ( - '{cls}(username={username}, ' - 'password=<*masked*>, key={key}, keys={keys})'.format( - cls=self.__class__.__name__, - username=self.username, - key=_key, - keys=_keys) - ) - - def __str__(self): - return ( - '{cls} for {username}'.format( - cls=self.__class__.__name__, - username=self.username, - ) - ) - - -class _MemorizedSSH(type): - """Memorize metaclass for SSHClient - - This class implements caching and managing of SSHClient connections. - Class is not in public scope: all required interfaces is accessible throw - SSHClient classmethods. - - Main flow is: - SSHClient() -> check for cached connection and - - If exists the same: check for alive, reconnect if required and return - - If exists with different credentials: delete and continue processing - create new connection and cache on success - * Note: each invocation of SSHClient instance will return current dir to - the root of the current user home dir ("cd ~"). - It is necessary to avoid unpredictable behavior when the same - connection is used from different places. - If you need to enter some directory and execute command there, please - use the following approach: - cmd1 = "cd && " - cmd2 = "cd && " - - Close cached connections is allowed per-client and all stored: - connection will be closed, but still stored in cache for faster reconnect - - Clear cache is strictly not recommended: - from this moment all open connections should be managed manually, - duplicates is possible. - """ - __cache = {} - - def __call__( - cls, - host, port=22, - username=None, password=None, private_keys=None, - auth=None, verbose=True - ): - """Main memorize method: check for cached instance and return it - - :type host: str - :type port: int - :type username: str - :type password: str - :type private_keys: list - :type auth: SSHAuth - :rtype: SSHClient - """ - if (host, port) in cls.__cache: - key = host, port - if auth is None: - auth = SSHAuth( - username=username, password=password, keys=private_keys) - if hash((cls, host, port, auth)) == hash(cls.__cache[key]): - ssh = cls.__cache[key] - # noinspection PyBroadException - try: - ssh.execute('cd ~', timeout=5) - except BaseException: # Note: Do not change to lower level! - logger.debug('Reconnect {}'.format(ssh)) - ssh.reconnect() - return ssh - if sys.getrefcount(cls.__cache[key]) == 2: - # If we have only cache reference and temporary getrefcount - # reference: close connection before deletion - logger.debug('Closing {} as unused'.format(cls.__cache[key])) - cls.__cache[key].close() - del cls.__cache[key] - # noinspection PyArgumentList - return super( - _MemorizedSSH, cls).__call__( - host=host, port=port, - username=username, password=password, private_keys=private_keys, - auth=auth, verbose=verbose) - - @classmethod - def record(mcs, ssh): - """Record SSH client to cache - - :type ssh: SSHClient - """ - mcs.__cache[(ssh.hostname, ssh.port)] = ssh - - @classmethod - def clear_cache(mcs): - """Clear cached connections for initialize new instance on next call""" - n_count = 3 if six.PY3 else 4 - # PY3: cache, ssh, temporary - # PY4: cache, values mapping, ssh, temporary - for ssh in mcs.__cache.values(): - if sys.getrefcount(ssh) == n_count: - logger.debug('Closing {} as unused'.format(ssh)) - ssh.close() - mcs.__cache = {} - - @classmethod - def close_connections(mcs, hostname=None): - """Close connections for selected or all cached records - - :type hostname: str - """ - if hostname is None: - keys = [key for key, ssh in mcs.__cache.items() if ssh.is_alive] - else: - keys = [ - (host, port) - for (host, port), ssh - in mcs.__cache.items() if host == hostname and ssh.is_alive] - # raise ValueError(keys) - for key in keys: - mcs.__cache[key].close() - - -class SSHClient(six.with_metaclass(_MemorizedSSH, object)): - __slots__ = [ - '__hostname', '__port', '__auth', '__ssh', '__sftp', 'sudo_mode', - '__lock', '__verbose' - ] - - class __get_sudo(object): - """Context manager for call commands with sudo""" - def __init__(self, ssh, enforce=None): - """Context manager for call commands with sudo - - :type ssh: SSHClient - :type enforce: bool - """ - self.__ssh = ssh - self.__sudo_status = ssh.sudo_mode - self.__enforce = enforce - - def __enter__(self): - self.__sudo_status = self.__ssh.sudo_mode - if self.__enforce is not None: - self.__ssh.sudo_mode = self.__enforce - - def __exit__(self, exc_type, exc_val, exc_tb): - self.__ssh.sudo_mode = self.__sudo_status - - # noinspection PyPep8Naming - class get_sudo(__get_sudo): - """Context manager for call commands with sudo""" - - def __init__(self, ssh, enforce=True): - warnings.warn( - 'SSHClient.get_sudo(SSHClient()) is deprecated in favor of ' - 'SSHClient().sudo(enforce=...) , which is much more powerful.') - super(self.__class__, self).__init__(ssh=ssh, enforce=enforce) - - def __hash__(self): - return hash(( - self.__class__, - self.hostname, - self.port, - self.auth)) - - def __init__( - self, - host, port=22, - username=None, password=None, private_keys=None, - auth=None, verbose=True - ): - """SSHClient helper - - :type host: str - :type port: int - :type username: str - :type password: str - :type private_keys: list - :type auth: SSHAuth - :type verbose: bool, show additional error/warning messages - """ - self.__lock = threading.RLock() - - self.__hostname = host - self.__port = port - - self.sudo_mode = False - self.__ssh = paramiko.SSHClient() - self.__ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - self.__sftp = None - - self.__auth = auth if auth is None else auth.copy() - self.__verbose = verbose - - if auth is None: - msg = ( - 'SSHClient(host={host}, port={port}, username={username}): ' - 'initialization by username/password/private_keys ' - 'is deprecated in favor of SSHAuth usage. ' - 'Please update your code'.format( - host=host, port=port, username=username - )) - warnings.warn(msg, DeprecationWarning) - logger.debug(msg) - - self.__auth = SSHAuth( - username=username, - password=password, - keys=private_keys - ) - - self.__connect() - _MemorizedSSH.record(ssh=self) - if auth is None: - logger.info( - '{0}:{1}> SSHAuth was made from old style creds: ' - '{2}'.format(self.hostname, self.port, self.auth)) - - @property - def lock(self): - """Connection lock - - :rtype: threading.RLock - """ - return self.__lock - - @property - def auth(self): - """Internal authorisation object - - Attention: this public property is mainly for inheritance, - debug and information purposes. - Calls outside SSHClient and child classes is sign of incorrect design. - Change is completely disallowed. - - :rtype: SSHAuth - """ - return self.__auth - - @property - def hostname(self): - """Connected remote host name - - :rtype: str - """ - return self.__hostname - - @property - def host(self): - """Hostname access for backward compatibility - - :rtype: str - """ - warnings.warn( - 'host has been deprecated in favor of hostname', - DeprecationWarning - ) - return self.hostname - - @property - def port(self): - """Connected remote port number - - :rtype: int - """ - return self.__port - - @property - def is_alive(self): - """Paramiko status: ready to use|reconnect required - - :rtype: bool - """ - return self.__ssh.get_transport() is not None - - def __repr__(self): - return '{cls}(host={host}, port={port}, auth={auth!r})'.format( - cls=self.__class__.__name__, host=self.hostname, port=self.port, - auth=self.auth - ) - - def __str__(self): - return '{cls}(host={host}, port={port}) for user {user}'.format( - cls=self.__class__.__name__, host=self.hostname, port=self.port, - user=self.auth.username - ) - - @property - def _ssh(self): - """ssh client object getter for inheritance support only - - Attention: ssh client object creation and change - is allowed only by __init__ and reconnect call. - - :rtype: paramiko.SSHClient - """ - return self.__ssh - - @decorators.retry(paramiko.SSHException, count=3, delay=3) - def __connect(self): - """Main method for connection open""" - with self.lock: - self.auth.connect( - client=self.__ssh, - hostname=self.hostname, port=self.port, - log=self.__verbose) - - def __connect_sftp(self): - """SFTP connection opener""" - with self.lock: - try: - self.__sftp = self.__ssh.open_sftp() - except paramiko.SSHException: - logger.warning('SFTP enable failed! SSH only is accessible.') - - @property - def _sftp(self): - """SFTP channel access for inheritance - - :rtype: paramiko.sftp_client.SFTPClient - :raises: paramiko.SSHException - """ - if self.__sftp is not None: - return self.__sftp - logger.debug('SFTP is not connected, try to connect...') - self.__connect_sftp() - if self.__sftp is not None: - return self.__sftp - raise paramiko.SSHException('SFTP connection failed') - - def close(self): - """Close SSH and SFTP sessions""" - with self.lock: - # noinspection PyBroadException - try: - self.__ssh.close() - self.__sftp = None - except Exception: - logger.exception("Could not close ssh connection") - if self.__sftp is not None: - # noinspection PyBroadException - try: - self.__sftp.close() - except Exception: - logger.exception("Could not close sftp connection") - - @staticmethod - def clear(): - warnings.warn( - "clear is removed: use close() only if it mandatory: " - "it's automatically called on revert|shutdown|suspend|destroy", - DeprecationWarning - ) - - @classmethod - def _clear_cache(cls): - """Enforce clear memorized records""" - warnings.warn( - '_clear_cache() is dangerous and not recommended for normal use!', - Warning - ) - _MemorizedSSH.clear_cache() - - @classmethod - def close_connections(cls, hostname=None): - """Close cached connections: if hostname is not set, then close all - - :type hostname: str - """ - _MemorizedSSH.close_connections(hostname=hostname) - - def __del__(self): - """Destructor helper: close channel and threads BEFORE closing others - - Due to threading in paramiko, default destructor could generate asserts - on close, so we calling channel close before closing main ssh object. - """ - self.__ssh.close() - self.__sftp = None - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def reconnect(self): - """Reconnect SSH session""" - with self.lock: - self.close() - - self.__ssh = paramiko.SSHClient() - self.__ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - self.__connect() - - def sudo(self, enforce=None): - """Call contextmanager for sudo mode change - - :type enforce: bool - :param enforce: Enforce sudo enabled or disabled. By default: None - """ - return self.__get_sudo(ssh=self, enforce=enforce) - - def check_call( - self, - command, verbose=False, timeout=None, - error_info=None, - expected=None, raise_on_err=True, **kwargs): - """Execute command and check for return code - - :type command: str - :type verbose: bool - :type timeout: int - :type error_info: str - :type expected: list - :type raise_on_err: bool - :rtype: ExecResult - :raises: DevopsCalledProcessError - """ - if expected is None: - expected = [proc_enums.ExitCodes.EX_OK] - else: - expected = [ - proc_enums.ExitCodes(code) - if ( - isinstance(code, int) and - code in proc_enums.ExitCodes.__members__.values()) - else code - for code in expected - ] - ret = self.execute(command, verbose, timeout, **kwargs) - if ret['exit_code'] not in expected: - message = ( - log_templates.CMD_UNEXPECTED_EXIT_CODE.format( - append=error_info + '\n' if error_info else '', - cmd=command, - code=ret['exit_code'], - expected=expected, - )) - logger.error(message) - if raise_on_err: - raise error.DevopsCalledProcessError( - command, ret['exit_code'], - expected=expected, - stdout=ret['stdout_brief'], - stderr=ret['stdout_brief']) - return ret - - def check_stderr( - self, - command, verbose=False, timeout=None, - error_info=None, - raise_on_err=True, **kwargs): - """Execute command expecting return code 0 and empty STDERR - - :type command: str - :type verbose: bool - :type timeout: int - :type error_info: str - :type raise_on_err: bool - :rtype: ExecResult - :raises: DevopsCalledProcessError - """ - ret = self.check_call( - command, verbose, timeout=timeout, - error_info=error_info, raise_on_err=raise_on_err, **kwargs) - if ret['stderr']: - message = ( - log_templates.CMD_UNEXPECTED_STDERR.format( - append=error_info + '\n' if error_info else '', - cmd=command, - code=ret['exit_code'], - )) - logger.error(message) - if raise_on_err: - raise error.DevopsCalledProcessError( - command, - ret['exit_code'], - stdout=ret['stdout_brief'], - stderr=ret['stdout_brief']) - return ret - - @classmethod - def execute_together( - cls, remotes, command, expected=None, raise_on_err=True, **kwargs): - """Execute command on multiple remotes in async mode - - :type remotes: list - :type command: str - :type expected: list - :type raise_on_err: bool - :raises: DevopsCalledProcessError - """ - if expected is None: - expected = [0] - futures = {} - errors = {} - for remote in set(remotes): # Use distinct remotes - chan, _, _, _ = remote.execute_async(command, **kwargs) - futures[remote] = chan - for remote, chan in futures.items(): - ret = chan.recv_exit_status() - chan.close() - if ret not in expected: - errors[remote.hostname] = ret - if errors and raise_on_err: - raise error.DevopsCalledProcessError(command, errors) - - @classmethod - def __exec_command( - cls, command, channel, stdout, stderr, timeout, verbose=False): - """Get exit status from channel with timeout - - :type command: str - :type channel: paramiko.channel.Channel - :type stdout: paramiko.channel.ChannelFile - :type stderr: paramiko.channel.ChannelFile - :type timeout: int - :type verbose: bool - :rtype: ExecResult - :raises: TimeoutError - """ - def poll_stream(src, verb_logger=None): - dst = [] - try: - for line in src: - dst.append(line) - if verb_logger is not None: - verb_logger( - line.decode('utf-8', - errors='backslashreplace').rstrip() - ) - except IOError: - pass - return dst - - def poll_streams(result, channel, stdout, stderr, verbose): - if channel.recv_ready(): - result.stdout += poll_stream( - src=stdout, - verb_logger=logger.info if verbose else logger.debug) - if channel.recv_stderr_ready(): - result.stderr += poll_stream( - src=stderr, - verb_logger=logger.error if verbose else logger.debug) - - @decorators.threaded(started=True) - def poll_pipes(stdout, stderr, result, stop, channel): - """Polling task for FIFO buffers - - :type stdout: paramiko.channel.ChannelFile - :type stderr: paramiko.channel.ChannelFile - :type result: ExecResult - :type stop: Event - :type channel: paramiko.channel.Channel - """ - - while not stop.isSet(): - time.sleep(0.1) - poll_streams( - result=result, - channel=channel, - stdout=stdout, - stderr=stderr, - verbose=verbose - ) - - if channel.status_event.is_set(): - result.exit_code = result.exit_code = channel.exit_status - - result.stdout += poll_stream( - src=stdout, - verb_logger=logger.info if verbose else logger.debug) - result.stderr += poll_stream( - src=stderr, - verb_logger=logger.error if verbose else logger.debug) - - stop.set() - - # channel.status_event.wait(timeout) - result = exec_result.ExecResult(cmd=command) - stop_event = threading.Event() - message = log_templates.CMD_EXEC.format(cmd=command.rstrip()) - if verbose: - logger.info(message) - else: - logger.debug(message) - - poll_pipes( - stdout=stdout, - stderr=stderr, - result=result, - stop=stop_event, - channel=channel - ) - - stop_event.wait(timeout) - - # Process closed? - if stop_event.isSet(): - stop_event.clear() - channel.close() - return result - - stop_event.set() - channel.close() - wait_err_msg = log_templates.CMD_WAIT_ERROR.format( - cmd=command.rstrip(), - timeout=timeout) - output_brief_msg = ('\tSTDOUT:\n' - '{0}\n' - '\tSTDERR"\n' - '{1}'.format(result.stdout_brief, - result.stderr_brief)) - logger.debug(wait_err_msg) - raise error.TimeoutError(wait_err_msg + output_brief_msg) - - def execute(self, command, verbose=False, timeout=None, **kwargs): - """Execute command and wait for return code - - :type command: str - :type verbose: bool - :type timeout: int - :rtype: ExecResult - :raises: TimeoutError - """ - chan, _, stderr, stdout = self.execute_async(command, **kwargs) - - result = self.__exec_command( - command, chan, stdout, stderr, timeout, - verbose=verbose - ) - message = (log_templates.CMD_RESULT.format( - cmd=command.rstrip(), code=result.exit_code)) - if verbose: - logger.info(message) - else: - logger.debug(message) - return result - - def execute_async(self, command, get_pty=False): - """Execute command in async mode and return channel with IO objects - - :type command: str - :type get_pty: bool - :rtype: - tuple( - paramiko.Channel, - paramiko.ChannelFile, - paramiko.ChannelFile, - paramiko.ChannelFile - ) - """ - message = log_templates.CMD_EXEC.format(cmd=command.rstrip()) - logger.debug(message) - - chan = self._ssh.get_transport().open_session() - - if get_pty: - # Open PTY - chan.get_pty( - term='vt100', - width=80, height=24, - width_pixels=0, height_pixels=0 - ) - - stdin = chan.makefile('wb') - stdout = chan.makefile('rb') - stderr = chan.makefile_stderr('rb') - cmd = "{}\n".format(command) - if self.sudo_mode: - encoded_cmd = base64.b64encode(cmd.encode('utf-8')).decode('utf-8') - cmd = ("sudo -S bash -c 'eval \"$(base64 -d " - "<(echo \"{0}\"))\"'").format( - encoded_cmd - ) - chan.exec_command(cmd) - if stdout.channel.closed is False: - self.auth.enter_password(stdin) - stdin.flush() - else: - chan.exec_command(cmd) - return chan, stdin, stderr, stdout - - def execute_through_host( - self, - hostname, - cmd, - auth=None, - target_port=22, - timeout=None, - verbose=False - ): - """Execute command on remote host through currently connected host - - :type hostname: str - :type cmd: str - :type auth: SSHAuth - :type target_port: int - :type timeout: int - :type verbose: bool - :rtype: ExecResult - :raises: TimeoutError - """ - if auth is None: - auth = self.auth - - intermediate_channel = self._ssh.get_transport().open_channel( - kind='direct-tcpip', - dest_addr=(hostname, target_port), - src_addr=(self.hostname, 0)) - transport = paramiko.Transport(sock=intermediate_channel) - - # start client and authenticate transport - auth.connect(transport) - - # open ssh session - channel = transport.open_session() - - # Make proxy objects for read - stdout = channel.makefile('rb') - stderr = channel.makefile_stderr('rb') - - channel.exec_command(cmd) - - # noinspection PyDictCreation - result = self.__exec_command( - cmd, channel, stdout, stderr, timeout, verbose=verbose) - - intermediate_channel.close() - - return result - - def _path_esc(self, path): - """Escape space character in the path""" - if path: - return (path.replace(' ', '\ ')) - - def mkdir(self, path): - """run 'mkdir -p path' on remote - - :type path: str - """ - if self.exists(path): - return - logger.debug("Creating directory: {}".format(self._path_esc(path))) - # noinspection PyTypeChecker - self.execute("mkdir -p {}\n".format(self._path_esc(path))) - - def rm_rf(self, path): - """run 'rm -rf path' on remote - - :type path: str - """ - logger.debug("rm -rf {}".format(self._path_esc(path))) - # noinspection PyTypeChecker - self.execute("rm -rf {}".format(self._path_esc(path))) - - def open(self, path, mode='r'): - """Open file on remote using SFTP session - - :type path: str - :type mode: str - :return: file.open() stream - """ - return self._sftp.open(path, mode) - - def upload(self, source, target): - """Upload file(s) from source to target using SFTP session - - :type source: str - :type target: str - """ - logger.debug("Copying '%s' -> '%s'", source, target) - - if self.isdir(target): - target = posixpath.join(target, os.path.basename(source)) - - source = os.path.expanduser(source) - if not os.path.isdir(source): - self._sftp.put(source, target) - return - - for rootdir, _, files in os.walk(source): - targetdir = os.path.normpath( - os.path.join( - target, - os.path.relpath(rootdir, source))).replace("\\", "/") - - self.mkdir(targetdir) - - for entry in files: - local_path = os.path.join(rootdir, entry) - remote_path = posixpath.join(targetdir, entry) - if self.exists(remote_path): - self._sftp.unlink(remote_path) - self._sftp.put(local_path, remote_path) - - def download(self, destination, target): - """Download file(s) to target from destination - - :type destination: str - :type target: str - :rtype: bool - """ - logger.debug( - "Copying '%s' -> '%s' from remote to local host", - destination, target - ) - - if os.path.isdir(target): - target = posixpath.join(target, os.path.basename(destination)) - - if not self.isdir(destination): - if self.exists(destination): - self._sftp.get(destination, target) - else: - logger.debug( - "Can't download %s because it doesn't exist", destination - ) - else: - logger.debug( - "Can't download %s because it is a directory", destination - ) - return os.path.exists(target) - - def exists(self, path): - """Check for file existence using SFTP session - - :type path: str - :rtype: bool - """ - try: - self._sftp.lstat(path) - return True - except IOError: - return False - - def stat(self, path): - """Get stat info for path with following symlinks - - :type path: str - :rtype: paramiko.sftp_attr.SFTPAttributes - """ - return self._sftp.stat(path) - - def isfile(self, path): - """Check, that path is file using SFTP session - - :type path: str - :rtype: bool - """ - try: - attrs = self._sftp.lstat(path) - return attrs.st_mode & stat.S_IFREG != 0 - except IOError: - return False - - def isdir(self, path): - """Check, that path is directory using SFTP session - - :type path: str - :rtype: bool - """ - try: - attrs = self._sftp.lstat(path) - return attrs.st_mode & stat.S_IFDIR != 0 - except IOError: - return False - -__all__ = ['SSHAuth', 'SSHClient'] +warnings.warn( + 'helpers.ssh_client was deprecated in favor of external exec_helpers', + DeprecationWarning +) diff --git a/devops/helpers/subprocess_runner.py b/devops/helpers/subprocess_runner.py index 4ecc545e..5b71a639 100644 --- a/devops/helpers/subprocess_runner.py +++ b/devops/helpers/subprocess_runner.py @@ -12,275 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import absolute_import from __future__ import unicode_literals -import fcntl -import os -import select -import subprocess -import threading -import time +import warnings -import six - -from devops import error -from devops.helpers import decorators -from devops.helpers import exec_result -from devops.helpers import log_templates -from devops.helpers import metaclasses -from devops.helpers import proc_enums -from devops import logger +from exec_helpers import Subprocess -class Subprocess(six.with_metaclass(metaclasses.SingletonMeta, object)): - __lock = threading.RLock() +__all__ = ('Subprocess',) - def __init__(self): - """Subprocess helper with timeouts and lock-free FIFO - - For excluding race-conditions we allow to run 1 command simultaneously - """ - pass - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - @classmethod - def __exec_command(cls, command, cwd=None, env=None, timeout=None, - verbose=False): - """Command executor helper - - :type command: str - :type cwd: str - :type env: dict - :type timeout: int - :rtype: ExecResult - """ - def poll_stream(src, verb_logger=None): - dst = [] - try: - for line in src: - dst.append(line) - if verb_logger is not None: - verb_logger( - line.decode('utf-8', - errors='backslashreplace').rstrip() - ) - except IOError: - pass - return dst - - def poll_streams(result, stdout, stderr, verbose): - rlist, _, _ = select.select( - [stdout, stderr], - [], - []) - if rlist: - if stdout in rlist: - result.stdout += poll_stream( - src=stdout, - verb_logger=logger.info if verbose else logger.debug) - if stderr in rlist: - result.stderr += poll_stream( - src=stderr, - verb_logger=logger.error if verbose else logger.debug) - - @decorators.threaded(started=True) - def poll_pipes(proc, result, stop): - """Polling task for FIFO buffers - - :type proc: subprocess.Popen - :type result: ExecResult - :type stop: threading.Event - """ - # Get file descriptors for stdout and stderr streams - fd_stdout = proc.stdout.fileno() - fd_stderr = proc.stderr.fileno() - # Get flags of stdout and stderr streams - fl_stdout = fcntl.fcntl(fd_stdout, fcntl.F_GETFL) - fl_stderr = fcntl.fcntl(fd_stderr, fcntl.F_GETFL) - # Set nonblock mode for stdout and stderr streams - fcntl.fcntl(fd_stdout, fcntl.F_SETFL, fl_stdout | os.O_NONBLOCK) - fcntl.fcntl(fd_stderr, fcntl.F_SETFL, fl_stderr | os.O_NONBLOCK) - - while not stop.isSet(): - time.sleep(0.1) - poll_streams( - result=result, - stdout=proc.stdout, - stderr=proc.stderr, - verbose=verbose - ) - - proc.poll() - - if proc.returncode is not None: - result.exit_code = proc.returncode - result.stdout += poll_stream( - src=proc.stdout, - verb_logger=logger.info if verbose else logger.debug) - result.stderr += poll_stream( - src=proc.stderr, - verb_logger=logger.error if verbose else logger.debug) - - stop.set() - - # 1 Command per run - with cls.__lock: - result = exec_result.ExecResult(cmd=command) - stop_event = threading.Event() - message = log_templates.CMD_EXEC.format(cmd=command.rstrip()) - if verbose: - logger.info(message) - else: - logger.debug(message) - # Run - process = subprocess.Popen( - args=[command], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True, cwd=cwd, env=env, - universal_newlines=False) - - # Poll output - poll_pipes(process, result, stop_event) - # wait for process close - stop_event.wait(timeout) - - # Process closed? - if stop_event.isSet(): - stop_event.clear() - return result - # Kill not ended process and wait for close - try: - process.kill() # kill -9 - stop_event.wait(5) - - except OSError: - # Nothing to kill - logger.warning( - u"{!s} has been completed just after timeout: " - "please validate timeout.".format(command)) - - wait_err_msg = log_templates.CMD_WAIT_ERROR.format( - cmd=command.rstrip(), - timeout=timeout) - output_brief_msg = ('\tSTDOUT:\n' - '{0}\n' - '\tSTDERR"\n' - '{1}'.format(result.stdout_brief, - result.stderr_brief)) - logger.debug(wait_err_msg) - raise error.TimeoutError(wait_err_msg + output_brief_msg) - - @classmethod - def execute(cls, command, verbose=False, timeout=None, **kwargs): - """Execute command and wait for return code - - Timeout limitation: read tick is 100 ms. - - :type command: str - :type verbose: bool - :type timeout: int - :rtype: ExecResult - :raises: TimeoutError - """ - result = cls.__exec_command(command=command, timeout=timeout, - verbose=verbose, **kwargs) - message = log_templates.CMD_RESULT.format( - cmd=command, code=result.exit_code) - if verbose: - logger.info(message) - else: - logger.debug(message) - - return result - - @classmethod - def check_call( - cls, - command, verbose=False, timeout=None, - error_info=None, - expected=None, raise_on_err=True, **kwargs): - """Execute command and check for return code - - Timeout limitation: read tick is 100 ms. - - :type command: str - :type verbose: bool - :type timeout: int - :type error_info: str - :type expected: list - :type raise_on_err: bool - :rtype: ExecResult - :raises: DevopsCalledProcessError - """ - - if expected is None: - expected = [proc_enums.ExitCodes.EX_OK] - else: - expected = [ - proc_enums.ExitCodes(code) - if ( - isinstance(code, int) and - code in proc_enums.ExitCodes.__members__.values()) - else code - for code in expected - ] - ret = cls.execute(command, verbose, timeout, **kwargs) - if ret['exit_code'] not in expected: - message = ( - log_templates.CMD_UNEXPECTED_EXIT_CODE.format( - append=error_info + '\n' if error_info else '', - cmd=command, - code=ret['exit_code'], - expected=expected - )) - logger.error(message) - if raise_on_err: - raise error.DevopsCalledProcessError( - command, ret['exit_code'], - expected=expected, - stdout=ret['stdout_brief'], - stderr=ret['stderr_brief']) - return ret - - @classmethod - def check_stderr( - cls, - command, verbose=False, timeout=None, - error_info=None, - raise_on_err=True, **kwargs): - """Execute command expecting return code 0 and empty STDERR - - Timeout limitation: read tick is 100 ms. - - :type command: str - :type verbose: bool - :type timeout: int - :type error_info: str - :type raise_on_err: bool - :rtype: ExecResult - :raises: DevopsCalledProcessError - """ - ret = cls.check_call( - command, verbose, timeout=timeout, - error_info=error_info, raise_on_err=raise_on_err, **kwargs) - if ret['stderr']: - message = ( - log_templates.CMD_UNEXPECTED_STDERR.format( - append=error_info + '\n' if error_info else '', - cmd=command, - code=ret['exit_code'] - )) - logger.error(message) - if raise_on_err: - raise error.DevopsCalledProcessError( - command, ret['exit_code'], - stdout=ret['stdout_brief'], - stderr=ret['stderr_brief']) - return ret +warnings.warn( + 'helpers.subprocess_runner.Subprocess is deprecated ' + 'in favor of external exec_helpers.Subprocess', + DeprecationWarning +) diff --git a/devops/models/environment.py b/devops/models/environment.py index ce4e467f..f0346bad 100644 --- a/devops/models/environment.py +++ b/devops/models/environment.py @@ -18,13 +18,13 @@ import warnings from django.conf import settings from django.db import IntegrityError from django.db import models +import exec_helpers import netaddr import paramiko from devops import error from devops.helpers import decorators from devops.helpers import network as network_helpers -from devops.helpers import ssh_client from devops import logger from devops.models import base from devops.models import driver @@ -417,9 +417,9 @@ class Environment(base.BaseModel): from devops import client env = client.DevopsClient().get_env(self.name) - return ssh_client.SSHClient( + return exec_helpers.SSHClient( ip, - auth=ssh_client.SSHAuth( + auth=exec_helpers.SSHAuth( username=login, password=password, keys=env.get_private_keys())) @@ -435,9 +435,9 @@ class Environment(base.BaseModel): logger.warning('Loading of SSH key from file failed. Trying to use' ' SSH agent ...') keys = paramiko.Agent().get_keys() - return ssh_client.SSHClient( + return exec_helpers.SSHClient( ip, - auth=ssh_client.SSHAuth(keys=keys)) + auth=exec_helpers.SSHAuth(keys=keys)) # LEGACY, TO REMOVE (for fuel-qa compatibility) def nodes(self): # migrated from EnvironmentModel.nodes() diff --git a/devops/tests/client/test_environment.py b/devops/tests/client/test_environment.py index f6719067..6aa19df3 100644 --- a/devops/tests/client/test_environment.py +++ b/devops/tests/client/test_environment.py @@ -14,12 +14,13 @@ import mock +import exec_helpers + from devops.client import environment from devops.client import nailgun from devops import error from devops.helpers import helpers from devops.helpers import ntp -from devops.helpers import ssh_client from devops.tests.driver import driverless @@ -41,7 +42,7 @@ class TestDevopsEnvironment(driverless.DriverlessTestCase): self.wait_tcp_mock = self.patch( 'devops.helpers.helpers.wait_tcp', spec=helpers.wait_tcp) self.ssh_mock = self.patch( - 'devops.helpers.ssh_client.SSHClient', spec=ssh_client.SSHClient) + 'exec_helpers.SSHClient', spec=exec_helpers.SSHClient) self.nc_mock = self.patch( 'devops.client.nailgun.NailgunClient', spec=nailgun.NailgunClient) self.nc_mock_inst = self.nc_mock.return_value @@ -172,7 +173,7 @@ class TestDevopsEnvironment(driverless.DriverlessTestCase): assert remote is ssh self.ssh_mock.assert_called_once_with( '10.109.0.2', - auth=ssh_client.SSHAuth(username='root', password='r00tme')) + auth=exec_helpers.SSHAuth(username='root', password='r00tme')) self.wait_tcp_mock.assert_called_once_with( host='10.109.0.2', port=22, timeout=180, @@ -224,7 +225,7 @@ class TestDevopsEnvironment(driverless.DriverlessTestCase): self.ssh_mock.assert_called_once_with( '10.109.0.2', - auth=ssh_client.SSHAuth(username='root', password='r00tme')) + auth=exec_helpers.SSHAuth(username='root', password='r00tme')) assert ssh.isfile.call_count == 2 ssh.isfile.assert_any_call('/root/.ssh/id_rsa') ssh.isfile.assert_any_call('/root/.ssh/bootstrap.rsa') @@ -268,7 +269,7 @@ class TestDevopsEnvironment(driverless.DriverlessTestCase): assert remote is ssh self.ssh_mock.assert_called_with( '10.109.0.100', - auth=ssh_client.SSHAuth( + auth=exec_helpers.SSHAuth( username='root', password='r00tme', keys=keys)) diff --git a/devops/tests/helpers/test_cloud_image_settings.py b/devops/tests/helpers/test_cloud_image_settings.py index 24abd43c..59860cad 100644 --- a/devops/tests/helpers/test_cloud_image_settings.py +++ b/devops/tests/helpers/test_cloud_image_settings.py @@ -32,7 +32,7 @@ class TestCloudImageSettings(unittest.TestCase): def setUp(self): self.subprocess_mock = self.patch( - 'devops.helpers.subprocess_runner.Subprocess', autospec=True) + 'exec_helpers.Subprocess', autospec=True) self.os_mock = self.patch( 'devops.helpers.cloud_image_settings.os', autospec=True) @@ -93,7 +93,7 @@ class TestCloudImageSettings(unittest.TestCase): mock.call().__exit__(None, None, None), )) - self.subprocess_mock.check_call.assert_called_once_with( + self.subprocess_mock().check_call.assert_called_once_with( 'genisoimage -output /mydir/cloud_settings.iso ' '-volid cidata -joliet -rock /mydir/user-data ' '/mydir/meta-data') diff --git a/devops/tests/helpers/test_decorators.py b/devops/tests/helpers/test_decorators.py index 89346000..2f7ad507 100644 --- a/devops/tests/helpers/test_decorators.py +++ b/devops/tests/helpers/test_decorators.py @@ -14,8 +14,6 @@ from __future__ import unicode_literals -import logging -import threading import unittest import mock @@ -24,97 +22,6 @@ from devops import error from devops.helpers import decorators -class ThreadedTest(unittest.TestCase): - def test_add_basic(self): - @decorators.threaded - def func_test(): - pass - # pylint: disable=assignment-from-no-return - test_thread = func_test() - # pylint: enable=assignment-from-no-return - self.assertEqual(test_thread.name, 'Threaded func_test') - self.assertFalse(test_thread.daemon) - self.assertFalse(test_thread.isAlive()) - - def test_add_func(self): - @decorators.threaded() - def func_test(): - pass - - # pylint: disable=assignment-from-no-return - test_thread = func_test() - # pylint: enable=assignment-from-no-return - self.assertEqual(test_thread.name, 'Threaded func_test') - self.assertFalse(test_thread.daemon) - self.assertFalse(test_thread.isAlive()) - - def test_name(self): - @decorators.threaded(name='test name') - def func_test(): - pass - - # pylint: disable=assignment-from-no-return - test_thread = func_test() - # pylint: enable=assignment-from-no-return - self.assertEqual(test_thread.name, 'test name') - self.assertFalse(test_thread.daemon) - self.assertFalse(test_thread.isAlive()) - - def test_daemon(self): - @decorators.threaded(daemon=True) - def func_test(): - pass - - # pylint: disable=assignment-from-no-return - test_thread = func_test() - # pylint: enable=assignment-from-no-return - self.assertEqual(test_thread.name, 'Threaded func_test') - self.assertTrue(test_thread.daemon) - self.assertFalse(test_thread.isAlive()) - - @mock.patch('threading.Thread', autospec=True) - def test_started(self, thread): - @decorators.threaded(started=True) - def func_test(): - pass - - func_test() - - self.assertIn(mock.call().start(), thread.mock_calls) - - def test_args(self): - event = threading.Event() - data = [] - # pylint: disable=global-variable-not-assigned - global data - # pylint: enable=global-variable-not-assigned - - @decorators.threaded(started=True) - def func_test(add, evnt): - data.append(add) - evnt.set() - - func_test(1, event) - event.wait(3) - self.assertEqual(data, [1]) - - def test_kwargs(self): - event = threading.Event() - data = [] - # pylint: disable=global-variable-not-assigned - global data - # pylint: enable=global-variable-not-assigned - - @decorators.threaded(started=True) - def func_test(add, evnt): - data.append(add) - evnt.set() - - func_test(add=2, evnt=event) - event.wait(3) - self.assertEqual(data, [2]) - - class TestRetry(unittest.TestCase): def patch(self, *args, **kwargs): @@ -196,315 +103,6 @@ class TestRetry(unittest.TestCase): retry_dec('wrong') -class TestPrettyRepr(unittest.TestCase): - def test_simple(self): - self.assertEqual( - decorators.pretty_repr(True), repr(True) - ) - - def test_text(self): - self.assertEqual( - decorators.pretty_repr('Unicode text'), "u'''Unicode text'''" - ) - self.assertEqual( - decorators.pretty_repr(b'bytes text\x01'), "b'''bytes text\x01'''" - ) - - def test_iterable(self): - self.assertEqual( - decorators.pretty_repr([1, 2, 3]), - '\n[{nl:<5}1,{nl:<5}2,{nl:<5}3,\n]'.format(nl='\n') - ) - self.assertEqual( - decorators.pretty_repr((1, 2, 3)), - '\n({nl:<5}1,{nl:<5}2,{nl:<5}3,\n)'.format(nl='\n') - ) - res = decorators.pretty_repr({1, 2, 3}) - self.assertTrue( - res.startswith('\n{') and res.endswith('\n}') - ) - - def test_dict(self): - self.assertEqual( - decorators.pretty_repr({1: 1, 2: 2, 33: 33}), - '\n{\n 1 : 1,\n 2 : 2,\n 33: 33,\n}' - ) - - def test_nested_dict(self): - test_obj = [ - { - 1: - { - 2: 3 - }, - 4: - { - 5: 6 - }, - }, - { - 7: 8, - 9: (10, 11) - }, - ( - 12, - 13, - ), - {14: {15: {16: {17: {18: {19: [20]}}}}}} - ] - exp_repr = ( - '\n[' - '\n {' - '\n 1: ' - '\n {' - '\n 2: 3,' - '\n },' - '\n 4: ' - '\n {' - '\n 5: 6,' - '\n },' - '\n },' - '\n {' - '\n 9: ' - '\n (' - '\n 10,' - '\n 11,' - '\n ),' - '\n 7: 8,' - '\n },' - '\n (' - '\n 12,' - '\n 13,' - '\n ),' - '\n {' - '\n 14: ' - '\n {' - '\n 15: {16: {17: {18: {19: [20]}}}},' - '\n },' - '\n },' - '\n]' - ) - self.assertEqual(decorators.pretty_repr(test_obj), exp_repr) - - -@mock.patch('devops.helpers.decorators.logger', autospec=True) -class TestLogWrap(unittest.TestCase): - def test_no_args(self, logger): - @decorators.logwrap - def func(): - return 'No args' - - result = func() - self.assertEqual(result, 'No args') - logger.assert_has_calls(( - mock.call.log( - level=logging.DEBUG, - msg="Calling: \n'func'()" - ), - mock.call.log( - level=logging.DEBUG, - msg="Done: 'func' with result:\n{}".format( - decorators.pretty_repr(result)) - ), - )) - - def test_args_simple(self, logger): - arg = 'test arg' - - @decorators.logwrap - def func(tst): - return tst - - result = func(arg) - self.assertEqual(result, arg) - logger.assert_has_calls(( - mock.call.log( - level=logging.DEBUG, - msg="Calling: \n'func'(\n 'tst'={},\n)".format( - decorators.pretty_repr(arg, indent=8, no_indent_start=True) - ) - ), - mock.call.log( - level=logging.DEBUG, - msg="Done: 'func' with result:\n{}".format( - decorators.pretty_repr(result)) - ), - )) - - def test_args_defaults(self, logger): - arg = 'test arg' - - @decorators.logwrap - def func(tst=arg): - return tst - - result = func() - self.assertEqual(result, arg) - logger.assert_has_calls(( - mock.call.log( - level=logging.DEBUG, - msg="Calling: \n'func'(\n 'tst'={},\n)".format( - decorators.pretty_repr(arg, indent=8, - no_indent_start=True)) - ), - mock.call.log( - level=logging.DEBUG, - msg="Done: 'func' with result:\n{}".format( - decorators.pretty_repr(result)) - ), - )) - - def test_args_complex(self, logger): - string = 'string' - dictionary = {'key': 'dictionary'} - - @decorators.logwrap - def func(param_string, param_dictionary): - return param_string, param_dictionary - - result = func(string, dictionary) - self.assertEqual(result, (string, dictionary)) - # raise ValueError(logger.mock_calls) - logger.assert_has_calls(( - mock.call.log( - level=logging.DEBUG, - msg="Calling: \n'func'(" - "\n 'param_string'={string}," - "\n 'param_dictionary'={dictionary},\n)".format( - string=decorators.pretty_repr( - string, - indent=8, no_indent_start=True), - dictionary=decorators.pretty_repr( - dictionary, - indent=8, no_indent_start=True) - ) - ), - mock.call.log( - level=logging.DEBUG, - msg="Done: 'func' with result:\n{}".format( - decorators.pretty_repr(result)) - ), - )) - - def test_args_kwargs(self, logger): - targs = ['string1', 'string2'] - tkwargs = {'key': 'tkwargs'} - - @decorators.logwrap - def func(*args, **kwargs): - return tuple(args), kwargs - - result = func(*targs, **tkwargs) - self.assertEqual(result, (tuple(targs), tkwargs)) - # raise ValueError(logger.mock_calls) - logger.assert_has_calls(( - mock.call.log( - level=logging.DEBUG, - msg="Calling: \n'func'(" - "\n 'args'={args}," - "\n 'kwargs'={kwargs},\n)".format( - args=decorators.pretty_repr( - tuple(targs), - indent=8, no_indent_start=True), - kwargs=decorators.pretty_repr( - tkwargs, - indent=8, no_indent_start=True) - ) - ), - mock.call.log( - level=logging.DEBUG, - msg="Done: 'func' with result:\n{}".format( - decorators.pretty_repr(result)) - ), - )) - - def test_renamed_args_kwargs(self, logger): - arg = 'arg' - targs = ['string1', 'string2'] - tkwargs = {'key': 'tkwargs'} - - @decorators.logwrap - def func(arg, *positional, **named): - return arg, tuple(positional), named - - result = func(arg, *targs, **tkwargs) - self.assertEqual(result, (arg, tuple(targs), tkwargs)) - # raise ValueError(logger.mock_calls) - logger.assert_has_calls(( - mock.call.log( - level=logging.DEBUG, - msg="Calling: \n'func'(" - "\n 'arg'={arg}," - "\n 'positional'={args}," - "\n 'named'={kwargs},\n)".format( - arg=decorators.pretty_repr( - arg, - indent=8, no_indent_start=True), - args=decorators.pretty_repr( - tuple(targs), - indent=8, no_indent_start=True), - kwargs=decorators.pretty_repr( - tkwargs, - indent=8, no_indent_start=True) - ) - ), - mock.call.log( - level=logging.DEBUG, - msg="Done: 'func' with result:\n{}".format( - decorators.pretty_repr(result)) - ), - )) - - def test_negative(self, logger): - @decorators.logwrap - def func(): - raise ValueError('as expected') - - with self.assertRaises(ValueError): - func() - - logger.assert_has_calls(( - mock.call.log( - level=logging.DEBUG, - msg="Calling: \n'func'()" - ), - mock.call.log( - level=logging.ERROR, - msg="Failed: \n'func'()", - exc_info=True - ), - )) - - def test_negative_substitutions(self, logger): - new_logger = mock.Mock(spec=logging.Logger, name='logger') - log = mock.Mock(name='log') - new_logger.attach_mock(log, 'log') - - @decorators.logwrap( - log=new_logger, - log_level=logging.INFO, - exc_level=logging.WARNING - ) - def func(): - raise ValueError('as expected') - - with self.assertRaises(ValueError): - func() - - self.assertEqual(len(logger.mock_calls), 0) - log.assert_has_calls(( - mock.call( - level=logging.INFO, - msg="Calling: \n'func'()" - ), - mock.call( - level=logging.WARNING, - msg="Failed: \n'func'()", - exc_info=True - ), - )) - - class TestProcLock(unittest.TestCase): def patch(self, *args, **kwargs): diff --git a/devops/tests/helpers/test_exec_result.py b/devops/tests/helpers/test_exec_result.py deleted file mode 100644 index c6289bcb..00000000 --- a/devops/tests/helpers/test_exec_result.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2016 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import unicode_literals - -# pylint: disable=no-self-use - -import unittest - -import mock - -from devops import error -from devops.helpers import exec_result -from devops.helpers.proc_enums import ExitCodes -from devops.helpers.subprocess_runner import Subprocess - - -cmd = "ls -la | awk \'{print $1}\'" - - -# noinspection PyTypeChecker -class TestExecResult(unittest.TestCase): - @mock.patch('devops.helpers.exec_result.logger') - def test_create_minimal(self, logger): - """Test defaults""" - result = exec_result.ExecResult(cmd=cmd) - self.assertEqual(result.cmd, cmd) - self.assertEqual(result.cmd, result['cmd']) - self.assertEqual(result.stdout, []) - self.assertEqual(result.stdout, result['stdout']) - self.assertEqual(result.stderr, []) - self.assertEqual(result.stderr, result['stderr']) - self.assertEqual(result.stdout_bin, bytearray()) - self.assertEqual(result.stderr_bin, bytearray()) - self.assertEqual(result.stdout_str, '') - self.assertEqual(result.stdout_str, result['stdout_str']) - self.assertEqual(result.stderr_str, '') - self.assertEqual(result.stderr_str, result['stderr_str']) - self.assertEqual(result.stdout_brief, '') - self.assertEqual(result.stdout_brief, result['stdout_brief']) - self.assertEqual(result.stderr_brief, '') - self.assertEqual(result.stderr_brief, result['stderr_brief']) - self.assertEqual(result.exit_code, ExitCodes.EX_INVALID) - self.assertEqual(result.exit_code, result['exit_code']) - self.assertEqual( - repr(result), - '{cls}(cmd={cmd!r}, stdout={stdout}, stderr={stderr}, ' - 'exit_code={exit_code!s})'.format( - cls=exec_result.ExecResult.__name__, - cmd=cmd, - stdout=[], - stderr=[], - exit_code=ExitCodes.EX_INVALID - ) - ) - self.assertEqual( - str(result), - "{cls}(\n\tcmd={cmd!r}," - "\n\t stdout=\n'{stdout_brief}'," - "\n\tstderr=\n'{stderr_brief}', " - '\n\texit_code={exit_code!s}\n)'.format( - cls=exec_result.ExecResult.__name__, - cmd=cmd, - stdout_brief='', - stderr_brief='', - exit_code=ExitCodes.EX_INVALID - ) - ) - - with self.assertRaises(IndexError): - # pylint: disable=pointless-statement - # noinspection PyStatementEffect - result['nonexistent'] - # pylint: enable=pointless-statement - - with self.assertRaises(error.DevopsError): - # pylint: disable=pointless-statement - # noinspection PyStatementEffect - result['stdout_json'] - # pylint: enable=pointless-statement - logger.assert_has_calls(( - mock.call.exception( - "{cmd} stdout is not valid json:\n" - "{stdout_str!r}\n".format( - cmd=cmd, - stdout_str='')), - )) - self.assertIsNone(result['stdout_yaml']) - - self.assertEqual( - hash(result), - hash((exec_result.ExecResult, cmd, '', '', ExitCodes.EX_INVALID)) - ) - - @mock.patch('devops.helpers.exec_result.logger', autospec=True) - def test_not_implemented(self, logger): - """Test assertion on non implemented deserializer""" - result = exec_result.ExecResult(cmd=cmd) - deserialize = getattr(result, '_ExecResult__deserialize') - with self.assertRaises(error.DevopsNotImplementedError): - deserialize('tst') - logger.assert_has_calls(( - mock.call.error( - '{fmt} deserialize target is not implemented'.format( - fmt='tst')), - )) - - def test_setters(self): - result = exec_result.ExecResult(cmd=cmd) - self.assertEqual(result.exit_code, ExitCodes.EX_INVALID) - result.exit_code = 0 - self.assertEqual(result.exit_code, 0) - self.assertEqual(result.exit_code, result['exit_code']) - - tst_stdout = [ - b'Test\n', - b'long\n', - b'stdout\n', - b'data\n', - b' \n', - b'5\n', - b'6\n', - b'7\n', - b'8\n', - b'end!\n' - ] - - tst_stderr = [b'test\n'] * 10 - - result['stdout'] = tst_stdout - self.assertEqual(result.stdout, tst_stdout) - self.assertEqual(result.stdout, result['stdout']) - - result['stderr'] = tst_stderr - self.assertEqual(result.stderr, tst_stderr) - self.assertEqual(result.stderr, result['stderr']) - - with self.assertRaises(TypeError): - result.exit_code = 'code' - - with self.assertRaises(error.DevopsError): - result['stdout_brief'] = 'test' - - with self.assertRaises(IndexError): - result['test'] = True - - with self.assertRaises(TypeError): - result.stdout = 'stdout' - - self.assertEqual(result.stdout, tst_stdout) - - with self.assertRaises(TypeError): - result.stderr = 'stderr' - - self.assertEqual(result.stderr, tst_stderr) - - self.assertEqual(result.stdout_bin, bytearray(b''.join(tst_stdout))) - self.assertEqual(result.stderr_bin, bytearray(b''.join(tst_stderr))) - - stdout_br = tst_stdout[:3] + [b'...\n'] + tst_stdout[-3:] - stderr_br = tst_stderr[:3] + [b'...\n'] + tst_stderr[-3:] - - stdout_brief = b''.join(stdout_br).strip().decode(encoding='utf-8') - stderr_brief = b''.join(stderr_br).strip().decode(encoding='utf-8') - - self.assertEqual(result.stdout_brief, stdout_brief) - self.assertEqual(result.stderr_brief, stderr_brief) - - def test_json(self): - result = exec_result.ExecResult('test', stdout=[b'{"test": true}']) - self.assertEqual(result.stdout_json, {'test': True}) - - @mock.patch('devops.helpers.exec_result.logger', autospec=True) - def test_deprecations(self, logger): - result = exec_result.ExecResult('test', stdout=[b'{"test": true}']) - for deprecated in ('stdout_json', 'stdout_yaml'): - result['{}'.format(deprecated)] = {'test': False} - logger.assert_has_calls(( - mock.call.warning( - '{key} is read-only and calculated automatically'.format( - key='{}'.format(deprecated) - )), - )) - self.assertEqual(result[deprecated], {'test': True}) - logger.reset_mock() - - @mock.patch('devops.helpers.exec_result.logger', autospec=True) - def test_wrong_result(self, logger): - """Test logging exception if stdout if not a correct json""" - cmd = "ls -la | awk \'{print $1\}\'" - result = Subprocess.execute(command=cmd) - with self.assertRaises(error.DevopsError): - # pylint: disable=pointless-statement - # noinspection PyStatementEffect - result.stdout_json - # pylint: enable=pointless-statement - logger.assert_has_calls(( - mock.call.exception( - "{cmd} stdout is not valid json:\n" - "{stdout_str!r}\n".format( - cmd=cmd, - stdout_str='')), - )) - self.assertIsNone(result['stdout_yaml']) diff --git a/devops/tests/helpers/test_helpers.py b/devops/tests/helpers/test_helpers.py index 74fe8063..f340698a 100644 --- a/devops/tests/helpers/test_helpers.py +++ b/devops/tests/helpers/test_helpers.py @@ -19,6 +19,7 @@ import socket import unittest +import exec_helpers import mock # pylint: disable=redefined-builtin @@ -27,9 +28,7 @@ from six.moves import xrange # pylint: enable=redefined-builtin from devops import error -from devops.helpers import exec_result from devops.helpers import helpers -from devops.helpers import ssh_client class TestHelpersHelpers(unittest.TestCase): @@ -52,8 +51,8 @@ class TestHelpersHelpers(unittest.TestCase): for port in xrange(32000, 32100)]) @mock.patch( - 'devops.helpers.subprocess_runner.Subprocess.execute', - return_value=exec_result.ExecResult( + 'exec_helpers.Subprocess.execute', + return_value=exec_helpers.ExecResult( cmd="ping -c 1 -W '{timeout:d}' '{host:s}'".format( host='127.0.0.1', timeout=1, ), @@ -196,7 +195,7 @@ class TestHelpersHelpers(unittest.TestCase): helpers.wait_tcp(host, port, timeout) ping.assert_called_once_with(host=host, port=port) - @mock.patch('devops.helpers.ssh_client.SSHClient', autospec=True) + @mock.patch('exec_helpers.SSHClient', autospec=True) @mock.patch('devops.helpers.helpers.wait') def test_wait_ssh_cmd(self, wait, ssh): host = '127.0.0.1' @@ -210,7 +209,7 @@ class TestHelpersHelpers(unittest.TestCase): host, port, check_cmd, username, password, timeout) ssh.assert_called_once_with( host=host, port=port, - auth=ssh_client.SSHAuth(username=username, password=password) + auth=exec_helpers.SSHAuth(username=username, password=password) ) wait.assert_called_once() # Todo: cover ssh_client.execute diff --git a/devops/tests/helpers/test_ntp.py b/devops/tests/helpers/test_ntp.py index 38264712..c296e094 100644 --- a/devops/tests/helpers/test_ntp.py +++ b/devops/tests/helpers/test_ntp.py @@ -18,11 +18,11 @@ import unittest +import exec_helpers import mock from devops import error from devops.helpers import ntp -from devops.helpers import ssh_client class NtpTestCase(unittest.TestCase): @@ -34,18 +34,18 @@ class NtpTestCase(unittest.TestCase): return m def setUp(self): - self.remote_mock = mock.Mock(spec=ssh_client.SSHClient) + self.remote_mock = mock.Mock(spec=exec_helpers.SSHClient) self.remote_mock.__repr__ = mock.Mock(return_value='') self.wait_mock = self.patch('devops.helpers.helpers.wait') @staticmethod def make_exec_result(stdout, exit_code=0): - return { - 'exit_code': exit_code, - 'stderr': [], - 'stdout': stdout.splitlines(True), - } + return exec_helpers.ExecResult( + cmd='n/a', + exit_code=exit_code, + stdout=stdout.splitlines(True) + ) class TestNtpInitscript(NtpTestCase): @@ -97,7 +97,7 @@ class TestNtpInitscript(NtpTestCase): "find /etc/init.d/ -regex '/etc/init.d/ntp.?' -executable"), mock.call('ntpq -pn 127.0.0.1'), )) - assert peers == ['Line3\n', 'Line4\n'] + assert peers == ('Line3\n', 'Line4\n') def test_date(self): self.remote_mock.execute.side_effect = ( @@ -214,7 +214,7 @@ class TestNtpPacemaker(NtpTestCase): self.remote_mock.execute.assert_called_once_with( 'ip netns exec vrouter ntpq -pn 127.0.0.1') - assert peers == ['Line3\n', 'Line4\n'] + assert peers == ('Line3\n', 'Line4\n') class TestNtpSystemd(NtpTestCase): diff --git a/devops/tests/helpers/test_ssh_client.py b/devops/tests/helpers/test_ssh_client.py deleted file mode 100644 index c3ab0326..00000000 --- a/devops/tests/helpers/test_ssh_client.py +++ /dev/null @@ -1,1964 +0,0 @@ -# coding=utf-8 - -# Copyright 2016 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import unicode_literals - -# pylint: disable=no-self-use - -import base64 -import contextlib -from os import path -import posixpath -import stat -import unittest - -import mock -import paramiko -# noinspection PyUnresolvedReferences -from six.moves import cStringIO - -from devops import error -from devops.helpers import exec_result -from devops.helpers import ssh_client - - -def gen_private_keys(amount=1): - keys = [] - for _ in range(amount): - keys.append(paramiko.RSAKey.generate(1024)) - return keys - - -def gen_public_key(private_key=None): - if private_key is None: - private_key = paramiko.RSAKey.generate(1024) - return '{0} {1}'.format(private_key.get_name(), private_key.get_base64()) - - -class FakeStream(object): - def __init__(self, *args): - self.__src = list(args) - - def __iter__(self): - if len(self.__src) == 0: - raise IOError() - for _ in range(len(self.__src)): - yield self.__src.pop(0) - - -host = '127.0.0.1' -port = 22 -username = 'user' -password = 'pass' -private_keys = [] -command = 'ls ~\nline 2\nline 3\nline с кирилицей' -command_log = u"Executing command:\n{!s}\n".format(command.rstrip()) -stdout_list = [b' \n', b'2\n', b'3\n', b' \n'] -stderr_list = [b' \n', b'0\n', b'1\n', b' \n'] -encoded_cmd = base64.b64encode( - "{}\n".format(command).encode('utf-8') -).decode('utf-8') - - -# noinspection PyTypeChecker -class TestSSHAuth(unittest.TestCase): - def tearDown(self): - ssh_client.SSHClient._clear_cache() - - def init_checks(self, username=None, password=None, key=None, keys=None): - """shared positive init checks - - :type username: str - :type password: str - :type key: paramiko.RSAKey - :type keys: list - """ - auth = ssh_client.SSHAuth( - username=username, - password=password, - key=key, - keys=keys - ) - - int_keys = [None] - if key is not None: - int_keys.append(key) - if keys is not None: - for k in keys: - if k not in int_keys: - int_keys.append(k) - - self.assertEqual(auth.username, username) - with contextlib.closing(cStringIO()) as tgt: - auth.enter_password(tgt) - self.assertEqual(tgt.getvalue(), '{}\n'.format(password)) - self.assertEqual( - auth.public_key, - gen_public_key(key) if key is not None else None) - - _key = ( - None if auth.public_key is None else - ''.format(auth.public_key) - ) - _keys = [] - for k in int_keys: - if k == key: - continue - _keys.append( - ''.format( - gen_public_key(k)) if k is not None else None) - - self.assertEqual( - repr(auth), - "{cls}(" - "username={username}, " - "password=<*masked*>, " - "key={key}, " - "keys={keys})".format( - cls=ssh_client.SSHAuth.__name__, - username=auth.username, - key=_key, - keys=_keys - ) - ) - self.assertEqual( - str(auth), - '{cls} for {username}'.format( - cls=ssh_client.SSHAuth.__name__, - username=auth.username, - ) - ) - - def test_init_username_only(self): - self.init_checks( - username=username - ) - - def test_init_username_password(self): - self.init_checks( - username=username, - password=password - ) - - def test_init_username_key(self): - self.init_checks( - username=username, - key=gen_private_keys(1).pop() - ) - - def test_init_username_password_key(self): - self.init_checks( - username=username, - password=password, - key=gen_private_keys(1).pop() - ) - - def test_init_username_password_keys(self): - self.init_checks( - username=username, - password=password, - keys=gen_private_keys(2) - ) - - def test_init_username_password_key_keys(self): - self.init_checks( - username=username, - password=password, - key=gen_private_keys(1).pop(), - keys=gen_private_keys(2) - ) - - -# noinspection PyTypeChecker -@mock.patch('devops.helpers.ssh_client.logger', autospec=True) -@mock.patch( - 'paramiko.AutoAddPolicy', autospec=True, return_value='AutoAddPolicy') -@mock.patch('paramiko.SSHClient', autospec=True) -class TestSSHClientInit(unittest.TestCase): - def tearDown(self): - ssh_client.SSHClient._clear_cache() - - def init_checks( - self, - client, policy, logger, - host=None, port=22, - username=None, password=None, private_keys=None, - auth=None - ): - """shared checks for positive cases - - :type client: mock.Mock - :type policy: mock.Mock - :type logger: mock.Mock - :type host: str - :type port: int - :type username: str - :type password: str - :type private_keys: list - :type auth: ssh_client.SSHAuth - """ - _ssh = mock.call() - - ssh = ssh_client.SSHClient( - host=host, - port=port, - username=username, - password=password, - private_keys=private_keys, - auth=auth - ) - client.assert_called_once() - policy.assert_called_once() - - if auth is None: - if private_keys is None or len(private_keys) == 0: - logger.assert_has_calls(( - mock.call.debug( - 'SSHClient(' - 'host={host}, port={port}, username={username}): ' - 'initialization by username/password/private_keys ' - 'is deprecated in favor of SSHAuth usage. ' - 'Please update your code'.format( - host=host, port=port, username=username - )), - mock.call.info( - '{0}:{1}> SSHAuth was made from old style creds: ' - 'SSHAuth for {2}'.format(host, port, username)) - )) - else: - logger.assert_has_calls(( - mock.call.debug( - 'SSHClient(' - 'host={host}, port={port}, username={username}): ' - 'initialization by username/password/private_keys ' - 'is deprecated in favor of SSHAuth usage. ' - 'Please update your code'.format( - host=host, port=port, username=username - )), - mock.call.debug( - 'Main key has been updated, public key is: \n' - '{}'.format(ssh.auth.public_key)), - mock.call.info( - '{0}:{1}> SSHAuth was made from old style creds: ' - 'SSHAuth for {2}'.format(host, port, username)) - )) - else: - logger.assert_not_called() - - if auth is None: - if private_keys is None or len(private_keys) == 0: - pkey = None - expected_calls = [ - _ssh, - _ssh.set_missing_host_key_policy('AutoAddPolicy'), - _ssh.connect( - hostname=host, password=password, - pkey=pkey, - port=port, username=username), - ] - else: - pkey = private_keys[0] - expected_calls = [ - _ssh, - _ssh.set_missing_host_key_policy('AutoAddPolicy'), - _ssh.connect( - hostname=host, password=password, - pkey=None, - port=port, username=username), - _ssh.connect( - hostname=host, password=password, - pkey=pkey, - port=port, username=username), - ] - - self.assertIn(expected_calls, client.mock_calls) - - self.assertEqual( - ssh.auth, - ssh_client.SSHAuth( - username=username, - password=password, - keys=private_keys - ) - ) - else: - self.assertEqual(ssh.auth, auth) - - sftp = ssh._sftp - self.assertEqual(sftp, client().open_sftp()) - - self.assertEqual(ssh._ssh, client()) - - self.assertEqual(ssh.hostname, host) - self.assertEqual(ssh.port, port) - - self.assertEqual( - repr(ssh), - '{cls}(host={host}, port={port}, auth={auth!r})'.format( - cls=ssh.__class__.__name__, host=ssh.hostname, - port=ssh.port, - auth=ssh.auth - ) - ) - - def test_init_host(self, client, policy, logger): - """Test with host only set""" - self.init_checks( - client, policy, logger, - host=host) - - def test_init_alternate_port(self, client, policy, logger): - """Test with alternate port""" - self.init_checks( - client, policy, logger, - host=host, - port=2222 - ) - - def test_init_username(self, client, policy, logger): - """Test with username only set from creds""" - self.init_checks( - client, policy, logger, - host=host, - username=username - ) - - def test_init_username_password(self, client, policy, logger): - """Test with username and password set from creds""" - self.init_checks( - client, policy, logger, - host=host, - username=username, - password=password - ) - - def test_init_username_password_empty_keys(self, client, policy, logger): - """Test with username, password and empty keys set from creds""" - self.init_checks( - client, policy, logger, - host=host, - username=username, - password=password, - private_keys=[] - ) - - def test_init_username_single_key(self, client, policy, logger): - """Test with username and single key set from creds""" - connect = mock.Mock( - side_effect=[ - paramiko.AuthenticationException, mock.Mock() - ]) - _ssh = mock.Mock() - _ssh.attach_mock(connect, 'connect') - client.return_value = _ssh - - self.init_checks( - client, policy, logger, - host=host, - username=username, - private_keys=gen_private_keys(1) - ) - - def test_init_username_password_single_key(self, client, policy, logger): - """Test with username, password and single key set from creds""" - connect = mock.Mock( - side_effect=[ - paramiko.AuthenticationException, mock.Mock() - ]) - _ssh = mock.Mock() - _ssh.attach_mock(connect, 'connect') - client.return_value = _ssh - - self.init_checks( - client, policy, logger, - host=host, - username=username, - password=password, - private_keys=gen_private_keys(1) - ) - - def test_init_username_multiple_keys(self, client, policy, logger): - """Test with username and multiple keys set from creds""" - connect = mock.Mock( - side_effect=[ - paramiko.AuthenticationException, mock.Mock() - ]) - _ssh = mock.Mock() - _ssh.attach_mock(connect, 'connect') - client.return_value = _ssh - - self.init_checks( - client, policy, logger, - host=host, - username=username, - private_keys=gen_private_keys(2) - ) - - def test_init_username_password_multiple_keys( - self, client, policy, logger): - """Test with username, password and multiple keys set from creds""" - connect = mock.Mock( - side_effect=[ - paramiko.AuthenticationException, mock.Mock() - ]) - _ssh = mock.Mock() - _ssh.attach_mock(connect, 'connect') - client.return_value = _ssh - - connect = mock.Mock( - side_effect=[ - paramiko.AuthenticationException, mock.Mock() - ]) - _ssh = mock.Mock() - _ssh.attach_mock(connect, 'connect') - client.return_value = _ssh - - self.init_checks( - client, policy, logger, - host=host, - username=username, - password=password, - private_keys=gen_private_keys(2) - ) - - def test_init_auth(self, client, policy, logger): - self.init_checks( - client, policy, logger, - host=host, - auth=ssh_client.SSHAuth( - username=username, - password=password, - key=gen_private_keys(1).pop() - ) - ) - - def test_init_auth_break(self, client, policy, logger): - self.init_checks( - client, policy, logger, - host=host, - username='Invalid', - password='Invalid', - private_keys=gen_private_keys(1), - auth=ssh_client.SSHAuth( - username=username, - password=password, - key=gen_private_keys(1).pop() - ) - ) - - def test_init_context(self, client, policy, logger): - with ssh_client.SSHClient(host=host, auth=ssh_client.SSHAuth()) as ssh: - client.assert_called_once() - policy.assert_called_once() - - logger.assert_not_called() - - self.assertEqual(ssh.auth, ssh_client.SSHAuth()) - - sftp = ssh._sftp - self.assertEqual(sftp, client().open_sftp()) - - self.assertEqual(ssh._ssh, client()) - - self.assertEqual(ssh.hostname, host) - self.assertEqual(ssh.port, port) - - def test_init_clear_failed(self, client, policy, logger): - """Test reconnect - - :type client: mock.Mock - :type policy: mock.Mock - :type logger: mock.Mock - """ - _ssh = mock.Mock() - _ssh.attach_mock( - mock.Mock( - side_effect=[ - Exception('Mocked SSH close()'), - mock.Mock() - ]), - 'close') - _sftp = mock.Mock() - _sftp.attach_mock( - mock.Mock( - side_effect=[ - Exception('Mocked SFTP close()'), - mock.Mock() - ]), - 'close') - client.return_value = _ssh - _ssh.attach_mock(mock.Mock(return_value=_sftp), 'open_sftp') - - ssh = ssh_client.SSHClient(host=host, auth=ssh_client.SSHAuth()) - client.assert_called_once() - policy.assert_called_once() - - logger.assert_not_called() - - self.assertEqual(ssh.auth, ssh_client.SSHAuth()) - - sftp = ssh._sftp - self.assertEqual(sftp, _sftp) - - self.assertEqual(ssh._ssh, _ssh) - - self.assertEqual(ssh.hostname, host) - self.assertEqual(ssh.port, port) - - logger.reset_mock() - - ssh.close() - - logger.assert_has_calls(( - mock.call.exception('Could not close ssh connection'), - mock.call.exception('Could not close sftp connection'), - )) - - def test_init_reconnect(self, client, policy, logger): - """Test reconnect - - :type client: mock.Mock - :type policy: mock.Mock - :type logger: mock.Mock - """ - ssh = ssh_client.SSHClient(host=host, auth=ssh_client.SSHAuth()) - client.assert_called_once() - policy.assert_called_once() - - logger.assert_not_called() - - self.assertEqual(ssh.auth, ssh_client.SSHAuth()) - - sftp = ssh._sftp - self.assertEqual(sftp, client().open_sftp()) - - self.assertEqual(ssh._ssh, client()) - - client.reset_mock() - policy.reset_mock() - - self.assertEqual(ssh.hostname, host) - self.assertEqual(ssh.port, port) - - ssh.reconnect() - - _ssh = mock.call() - - expected_calls = [ - _ssh.close(), - _ssh, - _ssh.set_missing_host_key_policy('AutoAddPolicy'), - _ssh.connect( - hostname='127.0.0.1', - password=None, - pkey=None, - port=22, - username=None), - ] - self.assertIn( - expected_calls, - client.mock_calls - ) - - client.assert_called_once() - policy.assert_called_once() - - logger.assert_not_called() - - self.assertEqual(ssh.auth, ssh_client.SSHAuth()) - - sftp = ssh._sftp - self.assertEqual(sftp, client().open_sftp()) - - self.assertEqual(ssh._ssh, client()) - - @mock.patch('time.sleep', autospec=True) - def test_init_password_required(self, sleep, client, policy, logger): - connect = mock.Mock(side_effect=paramiko.PasswordRequiredException) - _ssh = mock.Mock() - _ssh.attach_mock(connect, 'connect') - client.return_value = _ssh - - with self.assertRaises(paramiko.PasswordRequiredException): - ssh_client.SSHClient(host=host, auth=ssh_client.SSHAuth()) - logger.assert_has_calls(( - mock.call.exception('No password has been set!'), - )) - - @mock.patch('time.sleep', autospec=True) - def test_init_password_broken(self, sleep, client, policy, logger): - connect = mock.Mock(side_effect=paramiko.PasswordRequiredException) - _ssh = mock.Mock() - _ssh.attach_mock(connect, 'connect') - client.return_value = _ssh - - with self.assertRaises(paramiko.PasswordRequiredException): - ssh_client.SSHClient( - host=host, auth=ssh_client.SSHAuth(password=password)) - - logger.assert_has_calls(( - mock.call.critical( - 'Unexpected PasswordRequiredException, ' - 'when password is set!' - ), - )) - - @mock.patch('time.sleep', autospec=True) - def test_init_auth_impossible_password( - self, sleep, client, policy, logger): - connect = mock.Mock(side_effect=paramiko.AuthenticationException) - - _ssh = mock.Mock() - _ssh.attach_mock(connect, 'connect') - client.return_value = _ssh - - with self.assertRaises(paramiko.AuthenticationException): - ssh_client.SSHClient( - host=host, auth=ssh_client.SSHAuth(password=password)) - - logger.assert_has_calls( - ( - mock.call.exception( - 'Connection using stored authentication info failed!'), - ) * 3 - ) - - @mock.patch('time.sleep', autospec=True) - def test_init_auth_impossible_key(self, sleep, client, policy, logger): - connect = mock.Mock(side_effect=paramiko.AuthenticationException) - - _ssh = mock.Mock() - _ssh.attach_mock(connect, 'connect') - client.return_value = _ssh - - with self.assertRaises(paramiko.AuthenticationException): - ssh_client.SSHClient( - host=host, - auth=ssh_client.SSHAuth(key=gen_private_keys(1).pop()) - ) - - logger.assert_has_calls( - ( - mock.call.exception( - 'Connection using stored authentication info failed!'), - ) * 3 - ) - - def test_init_auth_pass_no_key(self, client, policy, logger): - connect = mock.Mock( - side_effect=[ - paramiko.AuthenticationException, - mock.Mock() - ]) - - _ssh = mock.Mock() - _ssh.attach_mock(connect, 'connect') - client.return_value = _ssh - key = gen_private_keys(1).pop() - - ssh = ssh_client.SSHClient( - host=host, - auth=ssh_client.SSHAuth( - username=username, - password=password, - key=key - ) - ) - - client.assert_called_once() - policy.assert_called_once() - - logger.assert_has_calls(( - mock.call.debug( - 'Main key has been updated, public key is: \nNone'), - )) - - self.assertEqual( - ssh.auth, - ssh_client.SSHAuth( - username=username, - password=password, - keys=[key] - ) - ) - - sftp = ssh._sftp - self.assertEqual(sftp, client().open_sftp()) - - self.assertEqual(ssh._ssh, client()) - - @mock.patch('time.sleep', autospec=True) - def test_init_auth_brute_impossible(self, sleep, client, policy, logger): - connect = mock.Mock(side_effect=paramiko.AuthenticationException) - - _ssh = mock.Mock() - _ssh.attach_mock(connect, 'connect') - client.return_value = _ssh - - with self.assertRaises(paramiko.AuthenticationException): - ssh_client.SSHClient( - host=host, - username=username, - private_keys=gen_private_keys(2)) - - logger.assert_has_calls( - ( - mock.call.debug( - 'SSHClient(' - 'host={host}, port={port}, username={username}): ' - 'initialization by username/password/private_keys ' - 'is deprecated in favor of SSHAuth usage. ' - 'Please update your code'.format( - host=host, port=port, username=username - )), - ) + ( - mock.call.exception( - 'Connection using stored authentication info failed!'), - ) * 3 - ) - - def test_init_no_sftp(self, client, policy, logger): - open_sftp = mock.Mock(side_effect=paramiko.SSHException) - - _ssh = mock.Mock() - _ssh.attach_mock(open_sftp, 'open_sftp') - client.return_value = _ssh - - ssh = ssh_client.SSHClient( - host=host, auth=ssh_client.SSHAuth(password=password)) - - with self.assertRaises(paramiko.SSHException): - # pylint: disable=pointless-statement - # noinspection PyStatementEffect - ssh._sftp - # pylint: enable=pointless-statement - logger.assert_has_calls(( - mock.call.debug('SFTP is not connected, try to connect...'), - mock.call.warning( - 'SFTP enable failed! SSH only is accessible.'), - )) - - def test_init_sftp_repair(self, client, policy, logger): - _sftp = mock.Mock() - open_sftp = mock.Mock( - side_effect=[ - paramiko.SSHException, - _sftp, _sftp]) - - _ssh = mock.Mock() - _ssh.attach_mock(open_sftp, 'open_sftp') - client.return_value = _ssh - - ssh = ssh_client.SSHClient( - host=host, auth=ssh_client.SSHAuth(password=password)) - - with self.assertRaises(paramiko.SSHException): - # pylint: disable=pointless-statement - # noinspection PyStatementEffect - ssh._sftp - # pylint: enable=pointless-statement - - logger.reset_mock() - - sftp = ssh._sftp - self.assertEqual(sftp, open_sftp()) - logger.assert_has_calls(( - mock.call.debug('SFTP is not connected, try to connect...'), - )) - - @mock.patch('devops.helpers.exec_result.ExecResult', autospec=True) - def test_init_memorize( - self, - Result, - client, policy, logger): - port1 = 2222 - host1 = '127.0.0.2' - - # 1. Normal init - ssh01 = ssh_client.SSHClient(host=host) - ssh02 = ssh_client.SSHClient(host=host) - ssh11 = ssh_client.SSHClient(host=host, port=port1) - ssh12 = ssh_client.SSHClient(host=host, port=port1) - ssh21 = ssh_client.SSHClient(host=host1) - ssh22 = ssh_client.SSHClient(host=host1) - - self.assertTrue(ssh01 is ssh02) - self.assertTrue(ssh11 is ssh12) - self.assertTrue(ssh21 is ssh22) - self.assertFalse(ssh01 is ssh11) - self.assertFalse(ssh01 is ssh21) - self.assertFalse(ssh11 is ssh21) - - # 2. Close connections check - client.reset_mock() - ssh01.close_connections(ssh01.hostname) - client.assert_has_calls(( - mock.call().get_transport(), - mock.call().get_transport(), - mock.call().close(), - mock.call().close(), - )) - client.reset_mock() - ssh01.close_connections() - # Mock returns false-connected state, so we just count close calls - - client.assert_has_calls(( - mock.call().get_transport(), - mock.call().get_transport(), - mock.call().get_transport(), - mock.call().close(), - mock.call().close(), - mock.call().close(), - )) - - # change creds - ssh_client.SSHClient( - host=host, auth=ssh_client.SSHAuth(username=username)) - - # Change back: new connection differs from old with the same creds - ssh004 = ssh_client.SSHAuth(host) - self.assertFalse(ssh01 is ssh004) - - @mock.patch('warnings.warn') - def test_init_memorize_close_unused(self, warn, client, policy, logger): - ssh0 = ssh_client.SSHClient(host=host) - text = str(ssh0) - del ssh0 # remove reference - now it's cached and unused - client.reset_mock() - logger.reset_mock() - # New connection on the same host:port with different auth - ssh1 = ssh_client.SSHClient( - host=host, auth=ssh_client.SSHAuth(username=username)) - logger.assert_has_calls(( - mock.call.debug('Closing {} as unused'.format(text)), - )) - client.assert_has_calls(( - mock.call().close(), - )) - text = str(ssh1) - del ssh1 # remove reference - now it's cached and unused - client.reset_mock() - logger.reset_mock() - ssh_client.SSHClient._clear_cache() - logger.assert_has_calls(( - mock.call.debug('Closing {} as unused'.format(text)), - )) - client.assert_has_calls(( - mock.call().close(), - )) - - @mock.patch( - 'devops.helpers.ssh_client.SSHClient.execute') - def test_init_memorize_reconnect(self, execute, client, policy, logger): - execute.side_effect = paramiko.SSHException - ssh_client.SSHClient(host=host) - client.reset_mock() - policy.reset_mock() - logger.reset_mock() - ssh_client.SSHClient(host=host) - client.assert_called_once() - policy.assert_called_once() - - @mock.patch('warnings.warn') - def test_init_clear(self, warn, client, policy, logger): - ssh01 = ssh_client.SSHClient(host=host, auth=ssh_client.SSHAuth()) - - # noinspection PyDeprecation - ssh01.clear() - warn.assert_called_once_with( - "clear is removed: use close() only if it mandatory: " - "it's automatically called on revert|shutdown|suspend|destroy", - DeprecationWarning - ) - - self.assertNotIn( - mock.call.close(), - client.mock_calls - ) - - @mock.patch('warnings.warn') - def test_deprecated_host(self, warn, client, policy, logger): - ssh01 = ssh_client.SSHClient(host=host, auth=ssh_client.SSHAuth()) - # noinspection PyDeprecation - self.assertEqual(ssh01.host, ssh01.hostname) - warn.assert_called_once_with( - 'host has been deprecated in favor of hostname', - DeprecationWarning - ) - - -@mock.patch('devops.helpers.ssh_client.logger', autospec=True) -@mock.patch( - 'paramiko.AutoAddPolicy', autospec=True, return_value='AutoAddPolicy') -@mock.patch('paramiko.SSHClient', autospec=True) -class TestExecute(unittest.TestCase): - def tearDown(self): - ssh_client.SSHClient._clear_cache() - - @staticmethod - def get_ssh(): - """SSHClient object builder for execution tests - - :rtype: ssh_client.SSHClient - """ - # noinspection PyTypeChecker - return ssh_client.SSHClient( - host=host, - port=port, - auth=ssh_client.SSHAuth( - username=username, - password=password - )) - - @staticmethod - def gen_cmd_result_log_message(result): - return (u"Command exit code '{code!s}':\n{cmd!s}\n" - .format(cmd=result.cmd.rstrip(), code=result.exit_code)) - - def test_execute_async(self, client, policy, logger): - chan = mock.Mock() - open_session = mock.Mock(return_value=chan) - transport = mock.Mock() - transport.attach_mock(open_session, 'open_session') - get_transport = mock.Mock(return_value=transport) - _ssh = mock.Mock() - _ssh.attach_mock(get_transport, 'get_transport') - client.return_value = _ssh - - ssh = self.get_ssh() - - # noinspection PyTypeChecker - result = ssh.execute_async(command=command) - get_transport.assert_called_once() - open_session.assert_called_once() - - self.assertIn(chan, result) - chan.assert_has_calls(( - mock.call.makefile('wb'), - mock.call.makefile('rb'), - mock.call.makefile_stderr('rb'), - mock.call.exec_command('{}\n'.format(command)) - )) - self.assertIn( - mock.call.debug(command_log), - logger.mock_calls - ) - - def test_execute_async_pty(self, client, policy, logger): - chan = mock.Mock() - open_session = mock.Mock(return_value=chan) - transport = mock.Mock() - transport.attach_mock(open_session, 'open_session') - get_transport = mock.Mock(return_value=transport) - _ssh = mock.Mock() - _ssh.attach_mock(get_transport, 'get_transport') - client.return_value = _ssh - - ssh = self.get_ssh() - - # noinspection PyTypeChecker - result = ssh.execute_async(command=command, get_pty=True) - get_transport.assert_called_once() - open_session.assert_called_once() - - self.assertIn(chan, result) - chan.assert_has_calls(( - mock.call.get_pty( - term='vt100', - width=80, height=24, - width_pixels=0, height_pixels=0 - ), - mock.call.makefile('wb'), - mock.call.makefile('rb'), - mock.call.makefile_stderr('rb'), - mock.call.exec_command('{}\n'.format(command)) - )) - self.assertIn( - mock.call.debug(command_log), - logger.mock_calls - ) - - def test_execute_async_sudo(self, client, policy, logger): - chan = mock.Mock() - open_session = mock.Mock(return_value=chan) - transport = mock.Mock() - transport.attach_mock(open_session, 'open_session') - get_transport = mock.Mock(return_value=transport) - _ssh = mock.Mock() - _ssh.attach_mock(get_transport, 'get_transport') - client.return_value = _ssh - - ssh = self.get_ssh() - ssh.sudo_mode = True - - # noinspection PyTypeChecker - result = ssh.execute_async(command=command) - get_transport.assert_called_once() - open_session.assert_called_once() - - self.assertIn(chan, result) - chan.assert_has_calls(( - mock.call.makefile('wb'), - mock.call.makefile('rb'), - mock.call.makefile_stderr('rb'), - mock.call.exec_command( - "sudo -S bash -c '" - "eval \"$(base64 -d <(echo \"{0}\"))\"'".format(encoded_cmd)) - )) - self.assertIn( - mock.call.debug(command_log), - logger.mock_calls - ) - - def test_execute_async_with_sudo_enforce(self, client, policy, logger): - chan = mock.Mock() - open_session = mock.Mock(return_value=chan) - transport = mock.Mock() - transport.attach_mock(open_session, 'open_session') - get_transport = mock.Mock(return_value=transport) - _ssh = mock.Mock() - _ssh.attach_mock(get_transport, 'get_transport') - client.return_value = _ssh - - ssh = self.get_ssh() - self.assertFalse(ssh.sudo_mode) - with ssh_client.SSHClient.sudo(ssh, enforce=True): - self.assertTrue(ssh.sudo_mode) - # noinspection PyTypeChecker - result = ssh.execute_async(command=command) - self.assertFalse(ssh.sudo_mode) - - get_transport.assert_called_once() - open_session.assert_called_once() - - self.assertIn(chan, result) - chan.assert_has_calls(( - mock.call.makefile('wb'), - mock.call.makefile('rb'), - mock.call.makefile_stderr('rb'), - mock.call.exec_command( - "sudo -S bash -c '" - "eval \"$(base64 -d <(echo \"{0}\"))\"'".format(encoded_cmd)) - )) - self.assertIn( - mock.call.debug(command_log), - logger.mock_calls - ) - - def test_execute_async_with_no_sudo_enforce(self, client, policy, logger): - chan = mock.Mock() - open_session = mock.Mock(return_value=chan) - transport = mock.Mock() - transport.attach_mock(open_session, 'open_session') - get_transport = mock.Mock(return_value=transport) - _ssh = mock.Mock() - _ssh.attach_mock(get_transport, 'get_transport') - client.return_value = _ssh - - ssh = self.get_ssh() - ssh.sudo_mode = True - - with ssh.sudo(enforce=False): - # noinspection PyTypeChecker - result = ssh.execute_async(command=command) - get_transport.assert_called_once() - open_session.assert_called_once() - - self.assertIn(chan, result) - chan.assert_has_calls(( - mock.call.makefile('wb'), - mock.call.makefile('rb'), - mock.call.makefile_stderr('rb'), - mock.call.exec_command('{}\n'.format(command)) - )) - self.assertIn( - mock.call.debug(command_log), - logger.mock_calls - ) - - def test_execute_async_with_none_enforce(self, client, policy, logger): - chan = mock.Mock() - open_session = mock.Mock(return_value=chan) - transport = mock.Mock() - transport.attach_mock(open_session, 'open_session') - get_transport = mock.Mock(return_value=transport) - _ssh = mock.Mock() - _ssh.attach_mock(get_transport, 'get_transport') - client.return_value = _ssh - - ssh = self.get_ssh() - ssh.sudo_mode = False - - with ssh.sudo(): - # noinspection PyTypeChecker - result = ssh.execute_async(command=command) - get_transport.assert_called_once() - open_session.assert_called_once() - - self.assertIn(chan, result) - chan.assert_has_calls(( - mock.call.makefile('wb'), - mock.call.makefile('rb'), - mock.call.makefile_stderr('rb'), - mock.call.exec_command('{}\n'.format(command)) - )) - self.assertIn( - mock.call.debug(command_log), - logger.mock_calls - ) - - @mock.patch('devops.helpers.ssh_client.SSHAuth.enter_password') - def test_execute_async_sudo_password( - self, enter_password, client, policy, logger): - stdin = mock.Mock(name='stdin') - stdout = mock.Mock(name='stdout') - stdout_channel = mock.Mock() - stdout_channel.configure_mock(closed=False) - stdout.attach_mock(stdout_channel, 'channel') - makefile = mock.Mock(side_effect=[stdin, stdout]) - chan = mock.Mock() - chan.attach_mock(makefile, 'makefile') - open_session = mock.Mock(return_value=chan) - transport = mock.Mock() - transport.attach_mock(open_session, 'open_session') - get_transport = mock.Mock(return_value=transport) - _ssh = mock.Mock() - _ssh.attach_mock(get_transport, 'get_transport') - client.return_value = _ssh - - ssh = self.get_ssh() - ssh.sudo_mode = True - - # noinspection PyTypeChecker - result = ssh.execute_async(command=command) - get_transport.assert_called_once() - open_session.assert_called_once() - # raise ValueError(closed.mock_calls) - enter_password.assert_called_once_with(stdin) - stdin.assert_has_calls((mock.call.flush(), )) - - self.assertIn(chan, result) - chan.assert_has_calls(( - mock.call.makefile('wb'), - mock.call.makefile('rb'), - mock.call.makefile_stderr('rb'), - mock.call.exec_command( - "sudo -S bash -c '" - "eval \"$(base64 -d <(echo \"{0}\"))\"'".format(encoded_cmd)) - )) - self.assertIn( - mock.call.debug(command_log), - logger.mock_calls - ) - - @staticmethod - def get_patched_execute_async_retval(ec=0, stderr_val=None): - """get patched execute_async retval - - :rtype: - Tuple( - mock.Mock, - str, - exec_result.ExecResult, - FakeStream, - FakeStream) - """ - out = stdout_list - err = stderr_list if stderr_val is None else [] - - stdout = FakeStream(*out) - stderr = FakeStream(*err) - - exit_code = ec - chan = mock.Mock() - recv_exit_status = mock.Mock(return_value=exit_code) - chan.attach_mock(recv_exit_status, 'recv_exit_status') - - wait = mock.Mock() - status_event = mock.Mock() - status_event.attach_mock(wait, 'wait') - chan.attach_mock(status_event, 'status_event') - chan.configure_mock(exit_status=exit_code) - - # noinspection PyTypeChecker - exp_result = exec_result.ExecResult( - cmd=command, - stderr=err, - stdout=out, - exit_code=ec - ) - - return chan, '', exp_result, stderr, stdout - - @mock.patch( - 'devops.helpers.ssh_client.SSHClient.execute_async') - def test_execute( - self, - execute_async, - client, policy, logger): - ( - chan, _stdin, exp_result, stderr, stdout - ) = self.get_patched_execute_async_retval() - is_set = mock.Mock(return_value=True) - chan.status_event.attach_mock(is_set, 'is_set') - - execute_async.return_value = chan, _stdin, stderr, stdout - - ssh = self.get_ssh() - - logger.reset_mock() - - # noinspection PyTypeChecker - result = ssh.execute(command=command, verbose=False) - - self.assertEqual( - result, - exp_result - ) - execute_async.assert_called_once_with(command) - chan.assert_has_calls((mock.call.status_event.is_set(), )) - message = self.gen_cmd_result_log_message(result) - logger.assert_has_calls([ - mock.call.debug(command_log), - ] + [ - mock.call.debug(str(x.rstrip().decode('utf-8'))) - for x in stdout_list - ] + [ - mock.call.debug(str(x.rstrip().decode('utf-8'))) - for x in stderr_list - ] + [ - mock.call.debug(message), - ]) - - @mock.patch( - 'devops.helpers.ssh_client.SSHClient.execute_async') - def test_execute_verbose( - self, - execute_async, - client, policy, logger): - ( - chan, _stdin, exp_result, stderr, stdout - ) = self.get_patched_execute_async_retval() - is_set = mock.Mock(return_value=True) - chan.status_event.attach_mock(is_set, 'is_set') - - execute_async.return_value = chan, _stdin, stderr, stdout - - ssh = self.get_ssh() - - logger.reset_mock() - - # noinspection PyTypeChecker - result = ssh.execute(command=command, verbose=True) - - self.assertEqual( - result, - exp_result - ) - execute_async.assert_called_once_with(command) - chan.assert_has_calls((mock.call.status_event.is_set(), )) - - message = self.gen_cmd_result_log_message(result) - logger.assert_has_calls([ - mock.call.info(command_log), - ] + [ - mock.call.info(str(x.rstrip().decode('utf-8'))) - for x in stdout_list - ] + [ - mock.call.error(str(x.rstrip().decode('utf-8'))) - for x in stderr_list - ] + [ - mock.call.info(message), - ]) - - @mock.patch('time.sleep', autospec=True) - @mock.patch( - 'devops.helpers.ssh_client.SSHClient.execute_async') - def test_execute_timeout( - self, - execute_async, sleep, - client, policy, logger): - ( - chan, _stdin, exp_result, stderr, stdout - ) = self.get_patched_execute_async_retval() - is_set = mock.Mock(return_value=True) - chan.status_event.attach_mock(is_set, 'is_set') - - execute_async.return_value = chan, _stdin, stderr, stdout - - ssh = self.get_ssh() - - logger.reset_mock() - - # noinspection PyTypeChecker - result = ssh.execute(command=command, verbose=False, timeout=1) - - self.assertEqual( - result, - exp_result - ) - execute_async.assert_called_once_with(command) - chan.assert_has_calls((mock.call.status_event.is_set(), )) - message = self.gen_cmd_result_log_message(result) - logger.assert_has_calls(( - mock.call.debug(message), - )) - - @mock.patch( - 'devops.helpers.ssh_client.SSHClient.execute_async') - def test_execute_timeout_fail( - self, - execute_async, - client, policy, logger): - ( - chan, _stdin, _, stderr, stdout - ) = self.get_patched_execute_async_retval() - is_set = mock.Mock(return_value=False) - chan.status_event.attach_mock(is_set, 'is_set') - chan.status_event.attach_mock(mock.Mock(), 'wait') - - execute_async.return_value = chan, _stdin, stderr, stdout - - ssh = self.get_ssh() - - logger.reset_mock() - - with self.assertRaises(error.TimeoutError): - # noinspection PyTypeChecker - ssh.execute(command=command, verbose=False, timeout=1) - - execute_async.assert_called_once_with(command) - chan.assert_has_calls((mock.call.status_event.is_set(), )) - - @mock.patch( - 'devops.helpers.ssh_client.SSHClient.execute_async') - def test_execute_together(self, execute_async, client, policy, logger): - ( - chan, _stdin, _, stderr, stdout - ) = self.get_patched_execute_async_retval() - execute_async.return_value = chan, _stdin, stderr, stdout - - host2 = '127.0.0.2' - - ssh = self.get_ssh() - # noinspection PyTypeChecker - ssh2 = ssh_client.SSHClient( - host=host2, - port=port, - auth=ssh_client.SSHAuth( - username=username, - password=password - )) - - remotes = [ssh, ssh2] - - # noinspection PyTypeChecker - ssh_client.SSHClient.execute_together( - remotes=remotes, command=command) - - self.assertEqual(execute_async.call_count, len(remotes)) - chan.assert_has_calls(( - mock.call.recv_exit_status(), - mock.call.close(), - mock.call.recv_exit_status(), - mock.call.close() - )) - - # noinspection PyTypeChecker - ssh_client.SSHClient.execute_together( - remotes=remotes, command=command, expected=[1], raise_on_err=False) - - with self.assertRaises(error.DevopsCalledProcessError): - # noinspection PyTypeChecker - ssh_client.SSHClient.execute_together( - remotes=remotes, command=command, expected=[1]) - - @mock.patch( - 'devops.helpers.ssh_client.SSHClient.execute') - def test_check_call(self, execute, client, policy, logger): - exit_code = 0 - return_value = { - 'stderr_str': '0\n1', - 'stdout_str': '2\n3', - 'stderr_brief': '0\n1', - 'stdout_brief': '2\n3', - 'exit_code': exit_code, - 'stderr': [b' \n', b'0\n', b'1\n', b' \n'], - 'stdout': [b' \n', b'2\n', b'3\n', b' \n']} - execute.return_value = return_value - - verbose = False - - ssh = self.get_ssh() - - # noinspection PyTypeChecker - result = ssh.check_call(command=command, verbose=verbose, timeout=None) - execute.assert_called_once_with(command, verbose, None) - self.assertEqual(result, return_value) - - exit_code = 1 - return_value['exit_code'] = exit_code - execute.reset_mock() - execute.return_value = return_value - with self.assertRaises(error.DevopsCalledProcessError): - # noinspection PyTypeChecker - ssh.check_call(command=command, verbose=verbose, timeout=None) - execute.assert_called_once_with(command, verbose, None) - - @mock.patch( - 'devops.helpers.ssh_client.SSHClient.execute') - def test_check_call_expected(self, execute, client, policy, logger): - exit_code = 0 - return_value = { - 'stderr_str': '0\n1', - 'stdout_str': '2\n3', - 'stderr_brief': '0\n1', - 'stdout_brief': '2\n3', - 'exit_code': exit_code, - 'stderr': [b' \n', b'0\n', b'1\n', b' \n'], - 'stdout': [b' \n', b'2\n', b'3\n', b' \n']} - execute.return_value = return_value - - verbose = False - - ssh = self.get_ssh() - - # noinspection PyTypeChecker - result = ssh.check_call( - command=command, verbose=verbose, timeout=None, expected=[0, 75]) - execute.assert_called_once_with(command, verbose, None) - self.assertEqual(result, return_value) - - exit_code = 1 - return_value['exit_code'] = exit_code - execute.reset_mock() - execute.return_value = return_value - with self.assertRaises(error.DevopsCalledProcessError): - # noinspection PyTypeChecker - ssh.check_call( - command=command, verbose=verbose, timeout=None, - expected=[0, 75] - ) - execute.assert_called_once_with(command, verbose, None) - - @mock.patch( - 'devops.helpers.ssh_client.SSHClient.check_call') - def test_check_stderr(self, check_call, client, policy, logger): - return_value = { - 'stderr_str': '', - 'stdout_str': '2\n3', - 'stderr_brief': '', - 'stdout_brief': '2\n3', - 'exit_code': 0, - 'stderr': [], - 'stdout': [b' \n', b'2\n', b'3\n', b' \n']} - check_call.return_value = return_value - - verbose = False - raise_on_err = True - - ssh = self.get_ssh() - - # noinspection PyTypeChecker - result = ssh.check_stderr( - command=command, verbose=verbose, timeout=None, - raise_on_err=raise_on_err) - check_call.assert_called_once_with( - command, verbose, timeout=None, - error_info=None, raise_on_err=raise_on_err) - self.assertEqual(result, return_value) - - return_value['stderr_str'] = '0\n1' - return_value['stderr'] = [b' \n', b'0\n', b'1\n', b' \n'] - - check_call.reset_mock() - check_call.return_value = return_value - with self.assertRaises(error.DevopsCalledProcessError): - # noinspection PyTypeChecker - ssh.check_stderr( - command=command, verbose=verbose, timeout=None, - raise_on_err=raise_on_err) - check_call.assert_called_once_with( - command, verbose, timeout=None, - error_info=None, raise_on_err=raise_on_err) - - -@mock.patch('devops.helpers.ssh_client.logger', autospec=True) -@mock.patch( - 'paramiko.AutoAddPolicy', autospec=True, return_value='AutoAddPolicy') -@mock.patch('paramiko.SSHClient', autospec=True) -@mock.patch('paramiko.Transport', autospec=True) -class TestExecuteThrowHost(unittest.TestCase): - def tearDown(self): - ssh_client.SSHClient._clear_cache() - - @staticmethod - def prepare_execute_through_host(transp, client, exit_code): - intermediate_channel = mock.Mock(name='intermediate_channel') - - open_channel = mock.Mock( - return_value=intermediate_channel, - name='open_channel' - ) - intermediate_transport = mock.Mock(name='intermediate_transport') - intermediate_transport.attach_mock(open_channel, 'open_channel') - get_transport = mock.Mock( - return_value=intermediate_transport, - name='get_transport' - ) - - _ssh = mock.Mock(neme='_ssh') - _ssh.attach_mock(get_transport, 'get_transport') - client.return_value = _ssh - - transport = mock.Mock(name='transport') - transp.return_value = transport - - recv_exit_status = mock.Mock(return_value=exit_code) - - channel = mock.Mock() - channel.attach_mock( - mock.Mock(return_value=FakeStream(b' \n', b'2\n', b'3\n', b' \n')), - 'makefile') - channel.attach_mock( - mock.Mock(return_value=FakeStream(b' \n', b'0\n', b'1\n', b' \n')), - 'makefile_stderr') - - channel.attach_mock(recv_exit_status, 'recv_exit_status') - open_session = mock.Mock(return_value=channel, name='open_session') - transport.attach_mock(open_session, 'open_session') - - wait = mock.Mock() - status_event = mock.Mock() - status_event.attach_mock(wait, 'wait') - channel.attach_mock(status_event, 'status_event') - channel.configure_mock(exit_status=exit_code) - - is_set = mock.Mock(return_value=True) - channel.status_event.attach_mock(is_set, 'is_set') - - return ( - open_session, transport, channel, get_transport, - open_channel, intermediate_channel - ) - - def test_execute_through_host_no_creds( - self, transp, client, policy, logger): - target = '127.0.0.2' - exit_code = 0 - - # noinspection PyTypeChecker - return_value = exec_result.ExecResult( - cmd=command, - stderr=[b' \n', b'0\n', b'1\n', b' \n'], - stdout=[b' \n', b'2\n', b'3\n', b' \n'], - exit_code=exit_code - ) - - ( - open_session, - transport, - channel, - get_transport, - open_channel, - intermediate_channel - ) = self.prepare_execute_through_host( - transp=transp, - client=client, - exit_code=exit_code) - - # noinspection PyTypeChecker - ssh = ssh_client.SSHClient( - host=host, - port=port, - auth=ssh_client.SSHAuth( - username=username, - password=password - )) - - # noinspection PyTypeChecker - result = ssh.execute_through_host(target, command) - self.assertEqual(result, return_value) - get_transport.assert_called_once() - open_channel.assert_called_once() - transp.assert_called_once_with(intermediate_channel) - open_session.assert_called_once() - transport.assert_has_calls(( - mock.call.connect(username=username, password=password, pkey=None), - mock.call.open_session() - )) - channel.assert_has_calls(( - mock.call.makefile('rb'), - mock.call.makefile_stderr('rb'), - mock.call.exec_command(command), - mock.call.recv_ready(), - mock.call.recv_stderr_ready(), - mock.call.status_event.is_set(), - mock.call.close() - )) - - def test_execute_through_host_auth( - self, transp, client, policy, logger): - _login = 'cirros' - _password = 'cubswin:)' - - target = '127.0.0.2' - exit_code = 0 - - # noinspection PyTypeChecker - return_value = exec_result.ExecResult( - cmd=command, - stderr=[b' \n', b'0\n', b'1\n', b' \n'], - stdout=[b' \n', b'2\n', b'3\n', b' \n'], - exit_code=exit_code - ) - - ( - open_session, transport, channel, get_transport, - open_channel, intermediate_channel - ) = self.prepare_execute_through_host( - transp, client, exit_code=exit_code) - - # noinspection PyTypeChecker - ssh = ssh_client.SSHClient( - host=host, - port=port, - auth=ssh_client.SSHAuth( - username=username, - password=password - )) - - # noinspection PyTypeChecker - result = ssh.execute_through_host( - target, command, - auth=ssh_client.SSHAuth(username=_login, password=_password)) - self.assertEqual(result, return_value) - get_transport.assert_called_once() - open_channel.assert_called_once() - transp.assert_called_once_with(intermediate_channel) - open_session.assert_called_once() - transport.assert_has_calls(( - mock.call.connect(username=_login, password=_password, pkey=None), - mock.call.open_session() - )) - channel.assert_has_calls(( - mock.call.makefile('rb'), - mock.call.makefile_stderr('rb'), - mock.call.exec_command(command), - mock.call.recv_ready(), - mock.call.recv_stderr_ready(), - mock.call.status_event.is_set(), - mock.call.close() - )) - - -@mock.patch('devops.helpers.ssh_client.logger', autospec=True) -@mock.patch( - 'paramiko.AutoAddPolicy', autospec=True, return_value='AutoAddPolicy') -@mock.patch('paramiko.SSHClient', autospec=True) -class TestSftp(unittest.TestCase): - def tearDown(self): - ssh_client.SSHClient._clear_cache() - - @staticmethod - def prepare_sftp_file_tests(client): - _ssh = mock.Mock() - client.return_value = _ssh - _sftp = mock.Mock() - open_sftp = mock.Mock(parent=_ssh, return_value=_sftp) - _ssh.attach_mock(open_sftp, 'open_sftp') - - # noinspection PyTypeChecker - ssh = ssh_client.SSHClient( - host=host, - port=port, - auth=ssh_client.SSHAuth( - username=username, - password=password - )) - return ssh, _sftp - - def test_exists(self, client, policy, logger): - ssh, _sftp = self.prepare_sftp_file_tests(client) - lstat = mock.Mock() - _sftp.attach_mock(lstat, 'lstat') - dst = '/etc' - - # noinspection PyTypeChecker - result = ssh.exists(dst) - self.assertTrue(result) - lstat.assert_called_once_with(dst) - - # Negative scenario - lstat.reset_mock() - lstat.side_effect = IOError - - # noinspection PyTypeChecker - result = ssh.exists(dst) - self.assertFalse(result) - lstat.assert_called_once_with(dst) - - def test_stat(self, client, policy, logger): - ssh, _sftp = self.prepare_sftp_file_tests(client) - stat = mock.Mock() - _sftp.attach_mock(stat, 'stat') - stat.return_value = paramiko.sftp_attr.SFTPAttributes() - stat.return_value.st_size = 0 - stat.return_value.st_uid = 0 - stat.return_value.st_gid = 0 - dst = '/etc/passwd' - - # noinspection PyTypeChecker - result = ssh.stat(dst) - self.assertEqual(result.st_size, 0) - self.assertEqual(result.st_uid, 0) - self.assertEqual(result.st_gid, 0) - - def test_isfile(self, client, policy, logger): - class Attrs(object): - def __init__(self, mode): - self.st_mode = mode - - ssh, _sftp = self.prepare_sftp_file_tests(client) - lstat = mock.Mock() - _sftp.attach_mock(lstat, 'lstat') - lstat.return_value = Attrs(stat.S_IFREG) - dst = '/etc/passwd' - - # noinspection PyTypeChecker - result = ssh.isfile(dst) - self.assertTrue(result) - lstat.assert_called_once_with(dst) - - # Negative scenario - lstat.reset_mock() - lstat.return_value = Attrs(stat.S_IFDIR) - - # noinspection PyTypeChecker - result = ssh.isfile(dst) - self.assertFalse(result) - lstat.assert_called_once_with(dst) - - lstat.reset_mock() - lstat.side_effect = IOError - - # noinspection PyTypeChecker - result = ssh.isfile(dst) - self.assertFalse(result) - lstat.assert_called_once_with(dst) - - def test_isdir(self, client, policy, logger): - class Attrs(object): - def __init__(self, mode): - self.st_mode = mode - - ssh, _sftp = self.prepare_sftp_file_tests(client) - lstat = mock.Mock() - _sftp.attach_mock(lstat, 'lstat') - lstat.return_value = Attrs(stat.S_IFDIR) - dst = '/etc/passwd' - - # noinspection PyTypeChecker - result = ssh.isdir(dst) - self.assertTrue(result) - lstat.assert_called_once_with(dst) - - # Negative scenario - lstat.reset_mock() - lstat.return_value = Attrs(stat.S_IFREG) - - # noinspection PyTypeChecker - result = ssh.isdir(dst) - self.assertFalse(result) - lstat.assert_called_once_with(dst) - - lstat.reset_mock() - lstat.side_effect = IOError - # noinspection PyTypeChecker - result = ssh.isdir(dst) - self.assertFalse(result) - lstat.assert_called_once_with(dst) - - @mock.patch('devops.helpers.ssh_client.SSHClient.exists') - @mock.patch('devops.helpers.ssh_client.SSHClient.execute') - def test_mkdir(self, execute, exists, client, policy, logger): - exists.side_effect = [False, True] - - dst = '~/tst dir' - escaped_dst = '~/tst\ dir' - - # noinspection PyTypeChecker - ssh = ssh_client.SSHClient( - host=host, - port=port, - auth=ssh_client.SSHAuth( - username=username, - password=password - )) - - # Path not exists - # noinspection PyTypeChecker - ssh.mkdir(dst) - exists.assert_called_once_with(dst) - execute.assert_called_once_with("mkdir -p {}\n".format(escaped_dst)) - - # Path exists - exists.reset_mock() - execute.reset_mock() - - # noinspection PyTypeChecker - ssh.mkdir(dst) - exists.assert_called_once_with(dst) - execute.assert_not_called() - - @mock.patch('devops.helpers.ssh_client.SSHClient.execute') - def test_rm_rf(self, execute, client, policy, logger): - dst = '~/tst' - - # noinspection PyTypeChecker - ssh = ssh_client.SSHClient( - host=host, - port=port, - auth=ssh_client.SSHAuth( - username=username, - password=password - )) - - # Path not exists - # noinspection PyTypeChecker - ssh.rm_rf(dst) - execute.assert_called_once_with("rm -rf {}".format(dst)) - - def test_open(self, client, policy, logger): - ssh, _sftp = self.prepare_sftp_file_tests(client) - fopen = mock.Mock(return_value=True) - _sftp.attach_mock(fopen, 'open') - - dst = '/etc/passwd' - mode = 'r' - # noinspection PyTypeChecker - result = ssh.open(dst) - fopen.assert_called_once_with(dst, mode) - self.assertTrue(result) - - @mock.patch('devops.helpers.ssh_client.SSHClient.exists') - @mock.patch('os.path.exists', autospec=True) - @mock.patch('devops.helpers.ssh_client.SSHClient.isdir') - @mock.patch('os.path.isdir', autospec=True) - def test_download( - self, - isdir, remote_isdir, exists, remote_exists, client, policy, logger - ): - ssh, _sftp = self.prepare_sftp_file_tests(client) - isdir.return_value = True - exists.side_effect = [True, False, False] - remote_isdir.side_effect = [False, False, True] - remote_exists.side_effect = [True, False, False] - - dst = '/etc/environment' - target = '/tmp/environment' - # noinspection PyTypeChecker - result = ssh.download(destination=dst, target=target) - self.assertTrue(result) - isdir.assert_called_once_with(target) - exists.assert_called_once_with(posixpath.join( - target, path.basename(dst))) - remote_isdir.assert_called_once_with(dst) - remote_exists.assert_called_once_with(dst) - _sftp.assert_has_calls(( - mock.call.get(dst, posixpath.join(target, path.basename(dst))), - )) - - # Negative scenarios - logger.reset_mock() - # noinspection PyTypeChecker - result = ssh.download(destination=dst, target=target) - logger.assert_has_calls(( - mock.call.debug( - "Copying '%s' -> '%s' from remote to local host", - '/etc/environment', - '/tmp/environment'), - mock.call.debug( - "Can't download %s because it doesn't exist", - '/etc/environment' - ), - )) - self.assertFalse(result) - - logger.reset_mock() - # noinspection PyTypeChecker - ssh.download(destination=dst, target=target) - logger.assert_has_calls(( - mock.call.debug( - "Copying '%s' -> '%s' from remote to local host", - '/etc/environment', - '/tmp/environment'), - mock.call.debug( - "Can't download %s because it is a directory", - '/etc/environment' - ), - )) - - @mock.patch('devops.helpers.ssh_client.SSHClient.isdir') - @mock.patch('os.path.isdir', autospec=True) - def test_upload_file( - self, isdir, remote_isdir, client, policy, logger - ): - ssh, _sftp = self.prepare_sftp_file_tests(client) - isdir.return_value = False - remote_isdir.return_value = False - target = '/etc/environment' - source = '/tmp/environment' - - # noinspection PyTypeChecker - ssh.upload(source=source, target=target) - isdir.assert_called_once_with(source) - remote_isdir.assert_called_once_with(target) - _sftp.assert_has_calls(( - mock.call.put(source, target), - )) - - @mock.patch('devops.helpers.ssh_client.SSHClient.exists') - @mock.patch('devops.helpers.ssh_client.SSHClient.mkdir') - @mock.patch('os.walk') - @mock.patch('devops.helpers.ssh_client.SSHClient.isdir') - @mock.patch('os.path.isdir', autospec=True) - def test_upload_dir( - self, - isdir, remote_isdir, walk, mkdir, exists, - client, policy, logger - ): - ssh, _sftp = self.prepare_sftp_file_tests(client) - isdir.return_value = True - remote_isdir.return_value = True - exists.return_value = True - target = '/etc' - source = '/tmp/bash' - filename = 'bashrc' - walk.return_value = (source, '', [filename]), - expected_path = posixpath.join(target, path.basename(source)) - expected_file = posixpath.join(expected_path, filename) - - # noinspection PyTypeChecker - ssh.upload(source=source, target=target) - isdir.assert_called_once_with(source) - remote_isdir.assert_called_once_with(target) - mkdir.assert_called_once_with(expected_path) - exists.assert_called_once_with(expected_file) - _sftp.assert_has_calls(( - mock.call.unlink(expected_file), - mock.call.put(posixpath.join(source, filename), expected_file), - )) diff --git a/devops/tests/helpers/test_subprocess_runner.py b/devops/tests/helpers/test_subprocess_runner.py deleted file mode 100644 index 9a2149e8..00000000 --- a/devops/tests/helpers/test_subprocess_runner.py +++ /dev/null @@ -1,252 +0,0 @@ -# coding=utf-8 - -# Copyright 2016 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import unicode_literals - -import subprocess -import unittest - -import mock - -from devops import error -from devops.helpers import exec_result -from devops.helpers import subprocess_runner - -command = 'ls ~\nline 2\nline 3\nline с кирилицей' -command_log = u"Executing command:\n{!s}\n".format(command.rstrip()) -stdout_list = [b' \n', b'2\n', b'3\n', b' \n'] -stderr_list = [b' \n', b'0\n', b'1\n', b' \n'] - - -class FakeFileStream(object): - def __init__(self, *args): - self.__src = list(args) - - def __iter__(self): - for _ in range(len(self.__src)): - yield self.__src.pop(0) - - def fileno(self): - return hash(tuple(self.__src)) - - -# TODO(AStepanov): Cover negative scenarios (timeout) - - -@mock.patch('devops.helpers.subprocess_runner.logger', autospec=True) -@mock.patch('select.select', autospec=True) -@mock.patch('fcntl.fcntl', autospec=True) -@mock.patch('subprocess.Popen', autospec=True, name='subprocess.Popen') -class TestSubprocessRunner(unittest.TestCase): - @staticmethod - def prepare_close(popen, stderr_val=None, ec=0): - stdout_lines = stdout_list - stderr_lines = stderr_list if stderr_val is None else [] - - stdout = FakeFileStream(*stdout_lines) - stderr = FakeFileStream(*stderr_lines) - - popen_obj = mock.Mock() - popen_obj.attach_mock(stdout, 'stdout') - popen_obj.attach_mock(stderr, 'stderr') - popen_obj.configure_mock(returncode=ec) - - popen.return_value = popen_obj - - # noinspection PyTypeChecker - exp_result = exec_result.ExecResult( - cmd=command, - stderr=stderr_lines, - stdout=stdout_lines, - exit_code=ec - ) - - return popen_obj, exp_result - - @staticmethod - def gen_cmd_result_log_message(result): - return (u"Command exit code '{code!s}':\n{cmd!s}\n" - .format(cmd=result.cmd.rstrip(), code=result.exit_code)) - - def test_call(self, popen, fcntl, select, logger): - popen_obj, exp_result = self.prepare_close(popen) - select.return_value = [popen_obj.stdout, popen_obj.stderr], [], [] - - runner = subprocess_runner.Subprocess() - - # noinspection PyTypeChecker - result = runner.execute(command) - self.assertEqual( - result, exp_result - - ) - popen.assert_has_calls(( - mock.call( - args=[command], - cwd=None, - env=None, - shell=True, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - universal_newlines=False), - )) - logger.assert_has_calls([ - mock.call.debug(command_log), - ] + [ - mock.call.debug(str(x.rstrip().decode('utf-8'))) - for x in stdout_list - ] + [ - mock.call.debug(str(x.rstrip().decode('utf-8'))) - for x in stderr_list - ] + [ - mock.call.debug(self.gen_cmd_result_log_message(result)), - ]) - self.assertIn( - mock.call.poll(), popen_obj.mock_calls - ) - - def test_call_verbose(self, popen, fcntl, select, logger): - popen_obj, _ = self.prepare_close(popen) - select.return_value = [popen_obj.stdout, popen_obj.stderr], [], [] - - runner = subprocess_runner.Subprocess() - - # noinspection PyTypeChecker - result = runner.execute(command, verbose=True) - - logger.assert_has_calls([ - mock.call.info(command_log), - ] + [ - mock.call.info(str(x.rstrip().decode('utf-8'))) - for x in stdout_list - ] + [ - mock.call.error(str(x.rstrip().decode('utf-8'))) - for x in stderr_list - ] + [ - mock.call.info(self.gen_cmd_result_log_message(result)), - ]) - - -@mock.patch('devops.helpers.subprocess_runner.logger', autospec=True) -class TestSubprocessRunnerHelpers(unittest.TestCase): - @mock.patch('devops.helpers.subprocess_runner.Subprocess.execute') - def test_check_call(self, execute, logger): - exit_code = 0 - return_value = { - 'stderr_str': '0\n1', - 'stdout_str': '2\n3', - 'stderr_brief': '0\n1', - 'stdout_brief': '2\n3', - 'exit_code': exit_code, - 'stderr': [b' \n', b'0\n', b'1\n', b' \n'], - 'stdout': [b' \n', b'2\n', b'3\n', b' \n']} - execute.return_value = return_value - - verbose = False - - runner = subprocess_runner.Subprocess() - - # noinspection PyTypeChecker - result = runner.check_call( - command=command, verbose=verbose, timeout=None) - execute.assert_called_once_with(command, verbose, None) - self.assertEqual(result, return_value) - - exit_code = 1 - return_value['exit_code'] = exit_code - execute.reset_mock() - execute.return_value = return_value - with self.assertRaises(error.DevopsCalledProcessError): - # noinspection PyTypeChecker - runner.check_call(command=command, verbose=verbose, timeout=None) - execute.assert_called_once_with(command, verbose, None) - - @mock.patch('devops.helpers.subprocess_runner.Subprocess.execute') - def test_check_call_expected(self, execute, logger): - exit_code = 0 - return_value = { - 'stderr_str': '0\n1', - 'stdout_str': '2\n3', - 'stderr_brief': '0\n1', - 'stdout_brief': '2\n3', - 'exit_code': exit_code, - 'stderr': [b' \n', b'0\n', b'1\n', b' \n'], - 'stdout': [b' \n', b'2\n', b'3\n', b' \n']} - execute.return_value = return_value - - verbose = False - - runner = subprocess_runner.Subprocess() - - # noinspection PyTypeChecker - result = runner.check_call( - command=command, verbose=verbose, timeout=None, expected=[0, 75]) - execute.assert_called_once_with(command, verbose, None) - self.assertEqual(result, return_value) - - exit_code = 1 - return_value['exit_code'] = exit_code - execute.reset_mock() - execute.return_value = return_value - with self.assertRaises(error.DevopsCalledProcessError): - # noinspection PyTypeChecker - runner.check_call( - command=command, verbose=verbose, timeout=None, - expected=[0, 75] - ) - execute.assert_called_once_with(command, verbose, None) - - @mock.patch('devops.helpers.subprocess_runner.Subprocess.check_call') - def test_check_stderr(self, check_call, logger): - return_value = { - 'stderr_str': '', - 'stdout_str': '2\n3', - 'stderr_brief': '', - 'stdout_brief': '2\n3', - 'exit_code': 0, - 'stderr': [], - 'stdout': [b' \n', b'2\n', b'3\n', b' \n']} - check_call.return_value = return_value - - verbose = False - raise_on_err = True - - runner = subprocess_runner.Subprocess() - - # noinspection PyTypeChecker - result = runner.check_stderr( - command=command, verbose=verbose, timeout=None, - raise_on_err=raise_on_err) - check_call.assert_called_once_with( - command, verbose, timeout=None, - error_info=None, raise_on_err=raise_on_err) - self.assertEqual(result, return_value) - - return_value['stderr_str'] = '0\n1' - return_value['stderr_brief'] = '0\n1' - return_value['stderr'] = [b' \n', b'0\n', b'1\n', b' \n'] - - check_call.reset_mock() - check_call.return_value = return_value - with self.assertRaises(error.DevopsCalledProcessError): - # noinspection PyTypeChecker - runner.check_stderr( - command=command, verbose=verbose, timeout=None, - raise_on_err=raise_on_err) - check_call.assert_called_once_with( - command, verbose, timeout=None, - error_info=None, raise_on_err=raise_on_err) diff --git a/devops/tests/models/node_ext/test_centos_master.py b/devops/tests/models/node_ext/test_centos_master.py index a8b606f6..a53d5509 100644 --- a/devops/tests/models/node_ext/test_centos_master.py +++ b/devops/tests/models/node_ext/test_centos_master.py @@ -82,8 +82,7 @@ class TestCentosMasterExt(LibvirtTestCase): self.wait_tcp_mock = self.patch( 'devops.helpers.helpers.wait_tcp') - @mock.patch( - 'devops.helpers.subprocess_runner.Subprocess', autospec=True) + @mock.patch('exec_helpers.Subprocess', autospec=True) @mock.patch('devops.driver.libvirt.libvirt_driver.uuid') @mock.patch('libvirt.virConnect.defineXML') @mock.patch.multiple(settings, CLOUD_IMAGE_DIR='/tmp/') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..56e77045 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,19 @@ +keystoneauth1>=2.1.0 +netaddr>=0.7.12,!=0.7.16 +paramiko>=1.16.0,!=2.0.1 +Django>=1.8,<1.9 +jsonfield +PyYAML>=3.1.0 +libvirt-python>=3.5.0,<4.1.0 +tabulate +six>=1.9.0 +python-dateutil>=2.4.2 +lxml +enum34; python_version < "3.4" +fasteners>=0.7.0 +virtualbmc + +tenacity +logwrap +exec-helpers>=3.1.4 +threaded \ No newline at end of file diff --git a/setup.py b/setup.py index 5f22fd7a..cae3bb37 100644 --- a/setup.py +++ b/setup.py @@ -12,11 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -import sys - import setuptools +with open("requirements.txt") as f: + required = f.read().splitlines() + + setuptools.setup( name='fuel-devops', version='3.0.5', @@ -39,22 +41,7 @@ setuptools.setup( ], data_files=[('bin', ['bin/dos_functions.sh'])], # Use magic in install_requires due to risk of old setuptools - install_requires=[ - 'keystoneauth1>=2.1.0', - 'netaddr>=0.7.12,!=0.7.16', - 'paramiko>=1.16.0,!=2.0.1', - 'Django>=1.8,<1.9', - 'jsonfield', - 'PyYAML>=3.1.0', - 'libvirt-python>=3.5.0,<4.1.0', - 'tabulate', - 'six>=1.9.0', - 'python-dateutil>=2.4.2', - 'lxml', - 'enum34' if sys.version_info.major == 2 else '', - 'fasteners>=0.7.0', - 'virtualbmc' - ], + install_requires=required, tests_require=[ 'pytest>=2.7.1', 'pytest-django >= 2.8.0', diff --git a/test-requirements.txt b/test-requirements.txt index 194f0815..d1d1390c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,5 @@ +-r requirements.txt sphinx<=1.4.9 mock>=1.2 pytest>=2.7.1 -pytest-django >= 2.8.0 +pytest-django >= 2.8.0 \ No newline at end of file diff --git a/tox.ini b/tox.ini index 67b95432..bda9cb97 100644 --- a/tox.ini +++ b/tox.ini @@ -5,13 +5,14 @@ [tox] minversion = 2.0 -envlist = pep8, py{27,35}, pylint, pylint-py{27,35}, cover, docs +envlist = pep8, py27, py3{5,6,7}, pylint, pylint-py{27,35}, cover, docs skipsdist = True skip_missing_interpreters = True [testenv] usedevelop = True +recreate = True passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY deps = -r{toxinidir}/test-requirements.txt