summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2017-03-23 12:52:18 +0000
committerGerrit Code Review <review@openstack.org>2017-03-23 12:52:18 +0000
commitd8c5b6b8b753ce9250b8fcf278f756a87b338e50 (patch)
tree395d21c207dfcccda6ca7f5654e29826fca4b1c9
parent0bcbdc402c64b33c81f45fb8fb7c13b943250ff2 (diff)
parent45e773d9dbd8ac9c85e3f11eb80b85edfcc12692 (diff)
Merge "Improve failover of SSH connections"
-rw-r--r--fuelweb_test/helpers/checkers.py4
-rw-r--r--fuelweb_test/helpers/ssh_manager.py164
-rw-r--r--fuelweb_test/models/environment.py1
-rw-r--r--fuelweb_test/tests/test_multipath_devices.py9
4 files changed, 106 insertions, 72 deletions
diff --git a/fuelweb_test/helpers/checkers.py b/fuelweb_test/helpers/checkers.py
index 2780b15..186f2ef 100644
--- a/fuelweb_test/helpers/checkers.py
+++ b/fuelweb_test/helpers/checkers.py
@@ -1466,8 +1466,8 @@ def check_package_version(ip, package_name, expected_version, condition='ge'):
1466 :param condition: predicate can be on of eq, ne, lt, le, ge, gt 1466 :param condition: predicate can be on of eq, ne, lt, le, ge, gt
1467 :return None: or raise UnexpectedExitCode 1467 :return None: or raise UnexpectedExitCode
1468 """ 1468 """
1469 cmd = "dpkg -s {0} " \ 1469 cmd = ("dpkg -s {0} "
1470 "| awk -F': ' '/Version/ {{print \$2}}'".format(package_name) 1470 "| awk -F': ' '/Version/ {{print $2}}'".format(package_name))
1471 logger.debug(cmd) 1471 logger.debug(cmd)
1472 result = ssh_manager.execute_on_remote( 1472 result = ssh_manager.execute_on_remote(
1473 ip, 1473 ip,
diff --git a/fuelweb_test/helpers/ssh_manager.py b/fuelweb_test/helpers/ssh_manager.py
index 143467f..63612c1 100644
--- a/fuelweb_test/helpers/ssh_manager.py
+++ b/fuelweb_test/helpers/ssh_manager.py
@@ -19,9 +19,10 @@ import traceback
19from warnings import warn 19from warnings import warn
20 20
21from devops.helpers.metaclasses import SingletonMeta 21from devops.helpers.metaclasses import SingletonMeta
22from devops.helpers.ssh_client import SSHAuth
22from devops.helpers.ssh_client import SSHClient 23from devops.helpers.ssh_client import SSHClient
23from paramiko import RSAKey 24from paramiko import RSAKey
24from paramiko.ssh_exception import AuthenticationException 25from paramiko import SSHException
25import six 26import six
26 27
27from fuelweb_test import logger 28from fuelweb_test import logger
@@ -67,8 +68,15 @@ class SSHManager(six.with_metaclass(SingletonMeta, object)):
67 self.slave_login = slave_login 68 self.slave_login = slave_login
68 self.__slave_password = slave_password 69 self.__slave_password = slave_password
69 70
70 @staticmethod 71 def _get_keys(self):
71 def _connect(remote): 72 keys = []
73 admin_remote = self.get_remote(self.admin_ip)
74 key_string = '/root/.ssh/id_rsa'
75 with admin_remote.open(key_string) as f:
76 keys.append(RSAKey.from_private_key(f))
77 return keys
78
79 def connect(self, remote):
72 """ Check if connection is stable and return this one 80 """ Check if connection is stable and return this one
73 81
74 :param remote: 82 :param remote:
@@ -80,99 +88,129 @@ class SSHManager(six.with_metaclass(SingletonMeta, object)):
80 seconds=5, 88 seconds=5,
81 error_message="Socket timeout! Forcing reconnection"): 89 error_message="Socket timeout! Forcing reconnection"):
82 remote.check_call("cd ~") 90 remote.check_call("cd ~")
83 except: 91 except Exception:
84 logger.debug(traceback.format_exc()) 92 logger.debug(traceback.format_exc())
85 logger.info('SSHManager: Check for current connection fails. ' 93 logger.debug('SSHManager: Check for current connection fails. '
86 'Trying to reconnect') 94 'Trying to reconnect')
87 remote.reconnect() 95 remote = self.reconnect(remote)
88 return remote 96 return remote
89 97
90 def _get_keys(self): 98 def reconnect(self, remote):
91 keys = [] 99 """ Reconnect to remote or update connection
92 admin_remote = self.get_remote(self.admin_ip)
93 key_string = '/root/.ssh/id_rsa'
94 with admin_remote.open(key_string) as f:
95 keys.append(RSAKey.from_private_key(f))
96 return keys
97 100
98 def get_remote(self, ip, port=22): 101 :param remote:
99 """ Function returns remote SSH connection to node by ip address 102 :return:
103 """
104 ip = remote.hostname
105 port = remote.port
106 try:
107 remote.reconnect()
108 except SSHException:
109 self.update_connection(ip=ip, port=port)
110 return self.connections[(ip, port)]
111
112 def init_remote(self, ip, port=22, custom_creds=None):
113 """ Initialise connection to remote
100 114
101 :param ip: IP of host 115 :param ip: IP of host
102 :type ip: str 116 :type ip: str
103 :param port: port for SSH 117 :param port: port for SSH
104 :type port: int 118 :type port: int
105 :rtype: SSHClient 119 :param custom_creds: custom creds
120 :type custom_creds: dict
106 """ 121 """
107 if (ip, port) not in self.connections: 122 logger.debug('SSH_MANAGER: Create new connection for '
108 logger.debug('SSH_MANAGER: Create new connection for ' 123 '{ip}:{port}'.format(ip=ip, port=port))
109 '{ip}:{port}'.format(ip=ip, port=port))
110 124
111 keys = self._get_keys() if ip != self.admin_ip else [] 125 keys = self._get_keys() if ip != self.admin_ip else []
112 if ip == self.admin_ip: 126 if ip == self.admin_ip:
127 ssh_client = SSHClient(
128 host=ip,
129 port=port,
130 auth=SSHAuth(
131 username=self.admin_login,
132 password=self.__admin_password,
133 keys=keys)
134 )
135 ssh_client.sudo_mode = SSH_FUEL_CREDENTIALS['sudo']
136 elif custom_creds:
137 ssh_client = SSHClient(
138 host=ip,
139 port=port,
140 auth=SSHAuth(**custom_creds))
141 else:
142 try:
113 ssh_client = SSHClient( 143 ssh_client = SSHClient(
114 host=ip, 144 host=ip,
115 port=port, 145 port=port,
116 username=self.admin_login, 146 auth=SSHAuth(
117 password=self.__admin_password,
118 private_keys=keys
119 )
120 ssh_client.sudo_mode = SSH_FUEL_CREDENTIALS['sudo']
121 else:
122 try:
123 ssh_client = SSHClient(
124 host=ip,
125 port=port,
126 username=self.slave_login, 147 username=self.slave_login,
127 password=self.__slave_password, 148 password=self.__slave_password,
128 private_keys=keys 149 keys=keys)
129 ) 150 )
130 except AuthenticationException: 151 except SSHException:
131 ssh_client = SSHClient( 152 ssh_client = SSHClient(
132 host=ip, 153 host=ip,
133 port=port, 154 port=port,
155 auth=SSHAuth(
134 username=self.slave_fallback_login, 156 username=self.slave_fallback_login,
135 password=self.__slave_password, 157 password=self.__slave_password,
136 private_keys=keys 158 keys=keys)
137 ) 159 )
138 ssh_client.sudo_mode = SSH_SLAVE_CREDENTIALS['sudo'] 160 ssh_client.sudo_mode = SSH_SLAVE_CREDENTIALS['sudo']
139 self.connections[(ip, port)] = ssh_client 161
140 logger.debug('SSH_MANAGER: Return existed connection for ' 162 self.connections[(ip, port)] = ssh_client
141 '{ip}:{port}'.format(ip=ip, port=port)) 163 logger.debug('SSH_MANAGER: New connection for '
164 '{ip}:{port} is created'.format(ip=ip, port=port))
165
166 def get_remote(self, ip, port=22):
167 """ Function returns remote SSH connection to node by ip address
168
169 :param ip: IP of host
170 :type ip: str
171 :param port: port for SSH
172 :type port: int
173 :rtype: SSHClient
174 """
175 if (ip, port) in self.connections:
176 logger.debug('SSH_MANAGER: Return existed connection for '
177 '{ip}:{port}'.format(ip=ip, port=port))
178 else:
179 self.init_remote(ip=ip, port=port)
142 logger.debug('SSH_MANAGER: Connections {0}'.format(self.connections)) 180 logger.debug('SSH_MANAGER: Connections {0}'.format(self.connections))
143 return self._connect(self.connections[(ip, port)]) 181 return self.connect(self.connections[(ip, port)])
144 182
145 def update_connection(self, ip, login=None, password=None, 183 def update_connection(self, ip, port=22, login=None, password=None,
146 keys=None, port=22): 184 keys=None):
147 """Update existed connection 185 """Update existed connection
148 186
149 :param ip: host ip string 187 :param ip: host ip string
188 :param port: ssh port int
150 :param login: login string 189 :param login: login string
151 :param password: password string 190 :param password: password string
152 :param keys: list of keys 191 :param keys: list of keys
153 :param port: ssh port int
154 :return: None 192 :return: None
155 """ 193 """
156 if (ip, port) in self.connections: 194 if (ip, port) in self.connections:
157 logger.info('SSH_MANAGER: Close connection for {ip}:{port}'.format( 195 logger.debug('SSH_MANAGER: Close connection for {ip}:{port}'
158 ip=ip, port=port)) 196 .format(ip=ip, port=port))
159 self.connections[(ip, port)].clear() 197 ssh_client = self.connections.pop((ip, port))
160 logger.info('SSH_MANAGER: Create new connection for ' 198 ssh_client.close()
161 '{ip}:{port}'.format(ip=ip, port=port)) 199 if login and (password or keys):
162 200 custom_creds = {
163 self.connections[(ip, port)] = SSHClient( 201 'username': login,
164 host=ip, 202 'password': password,
165 port=port, 203 'keys': keys
166 username=login, 204 }
167 password=password, 205 else:
168 private_keys=keys if keys is not None else [] 206 custom_creds = None
169 ) 207 self.init_remote(ip=ip, port=port, custom_creds=custom_creds)
170 208
171 def clean_all_connections(self): 209 def clean_all_connections(self):
172 for (ip, port), connection in self.connections.items(): 210 for (ip, port), connection in self.connections.items():
173 connection.clear() 211 connection.clear()
174 logger.info('SSH_MANAGER: Close connection for {ip}:{port}'.format( 212 logger.debug('SSH_MANAGER: Close connection for {ip}:{port}'
175 ip=ip, port=port)) 213 .format(ip=ip, port=port))
176 214
177 def execute(self, ip, cmd, port=22, sudo=None): 215 def execute(self, ip, cmd, port=22, sudo=None):
178 remote = self.get_remote(ip=ip, port=port) 216 remote = self.get_remote(ip=ip, port=port)
diff --git a/fuelweb_test/models/environment.py b/fuelweb_test/models/environment.py
index 08dd32f..3eeb915 100644
--- a/fuelweb_test/models/environment.py
+++ b/fuelweb_test/models/environment.py
@@ -377,6 +377,7 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
377 ) 377 )
378 self.ssh_manager.update_connection( 378 self.ssh_manager.update_connection(
379 ip=self.ssh_manager.admin_ip, 379 ip=self.ssh_manager.admin_ip,
380 port=22,
380 login=new_login, 381 login=new_login,
381 password=new_password 382 password=new_password
382 ) 383 )
diff --git a/fuelweb_test/tests/test_multipath_devices.py b/fuelweb_test/tests/test_multipath_devices.py
index 0d07aac..3fc927f 100644
--- a/fuelweb_test/tests/test_multipath_devices.py
+++ b/fuelweb_test/tests/test_multipath_devices.py
@@ -25,7 +25,6 @@ from fuelweb_test.settings import MULTIPATH
25from fuelweb_test.settings import MULTIPATH_TEMPLATE 25from fuelweb_test.settings import MULTIPATH_TEMPLATE
26from fuelweb_test.settings import NEUTRON_SEGMENT 26from fuelweb_test.settings import NEUTRON_SEGMENT
27from fuelweb_test.settings import SLAVE_MULTIPATH_DISKS_COUNT 27from fuelweb_test.settings import SLAVE_MULTIPATH_DISKS_COUNT
28from fuelweb_test.settings import SSH_FUEL_CREDENTIALS
29from fuelweb_test.settings import REPLACE_DEFAULT_REPOS 28from fuelweb_test.settings import REPLACE_DEFAULT_REPOS
30from fuelweb_test.settings import REPLACE_DEFAULT_REPOS_ONLY_ONCE 29from fuelweb_test.settings import REPLACE_DEFAULT_REPOS_ONLY_ONCE
31from fuelweb_test.tests import base_test_case 30from fuelweb_test.tests import base_test_case
@@ -53,9 +52,7 @@ class TestMultipath(base_test_case.TestBasic):
53 """ 52 """
54 cmd = "multipath -l -v2" 53 cmd = "multipath -l -v2"
55 54
56 ssh_manager.update_connection(ip, SSH_FUEL_CREDENTIALS['login'], 55 ssh_manager.update_connection(ip)
57 SSH_FUEL_CREDENTIALS['password'],
58 keys=ssh_manager._get_keys())
59 ssh_manager.get_remote(ip) 56 ssh_manager.get_remote(ip)
60 result = ssh_manager.execute_on_remote( 57 result = ssh_manager.execute_on_remote(
61 ip=ip, 58 ip=ip,
@@ -106,9 +103,7 @@ class TestMultipath(base_test_case.TestBasic):
106 """ 103 """
107 cmd = "lsblk -lo NAME,TYPE,MOUNTPOINT | grep '/$' | grep -c lvm" 104 cmd = "lsblk -lo NAME,TYPE,MOUNTPOINT | grep '/$' | grep -c lvm"
108 105
109 ssh_manager.update_connection(ip, SSH_FUEL_CREDENTIALS['login'], 106 ssh_manager.update_connection(ip)
110 SSH_FUEL_CREDENTIALS['password'],
111 keys=ssh_manager._get_keys())
112 ssh_manager.get_remote(ip) 107 ssh_manager.get_remote(ip)
113 result = ssh_manager.execute_on_remote( 108 result = ssh_manager.execute_on_remote(
114 ip=ip, 109 ip=ip,