diff options
author | Jenkins <jenkins@review.openstack.org> | 2017-03-23 12:52:18 +0000 |
---|---|---|
committer | Gerrit Code Review <review@openstack.org> | 2017-03-23 12:52:18 +0000 |
commit | d8c5b6b8b753ce9250b8fcf278f756a87b338e50 (patch) | |
tree | 395d21c207dfcccda6ca7f5654e29826fca4b1c9 | |
parent | 0bcbdc402c64b33c81f45fb8fb7c13b943250ff2 (diff) | |
parent | 45e773d9dbd8ac9c85e3f11eb80b85edfcc12692 (diff) |
Merge "Improve failover of SSH connections"
-rw-r--r-- | fuelweb_test/helpers/checkers.py | 4 | ||||
-rw-r--r-- | fuelweb_test/helpers/ssh_manager.py | 164 | ||||
-rw-r--r-- | fuelweb_test/models/environment.py | 1 | ||||
-rw-r--r-- | fuelweb_test/tests/test_multipath_devices.py | 9 |
4 files changed, 106 insertions, 72 deletions
diff --git a/fuelweb_test/helpers/checkers.py b/fuelweb_test/helpers/checkers.py index 2780b15..186f2ef 100644 --- a/fuelweb_test/helpers/checkers.py +++ b/fuelweb_test/helpers/checkers.py | |||
@@ -1466,8 +1466,8 @@ def check_package_version(ip, package_name, expected_version, condition='ge'): | |||
1466 | :param condition: predicate can be on of eq, ne, lt, le, ge, gt | 1466 | :param condition: predicate can be on of eq, ne, lt, le, ge, gt |
1467 | :return None: or raise UnexpectedExitCode | 1467 | :return None: or raise UnexpectedExitCode |
1468 | """ | 1468 | """ |
1469 | cmd = "dpkg -s {0} " \ | 1469 | cmd = ("dpkg -s {0} " |
1470 | "| awk -F': ' '/Version/ {{print \$2}}'".format(package_name) | 1470 | "| awk -F': ' '/Version/ {{print $2}}'".format(package_name)) |
1471 | logger.debug(cmd) | 1471 | logger.debug(cmd) |
1472 | result = ssh_manager.execute_on_remote( | 1472 | result = ssh_manager.execute_on_remote( |
1473 | ip, | 1473 | ip, |
diff --git a/fuelweb_test/helpers/ssh_manager.py b/fuelweb_test/helpers/ssh_manager.py index 143467f..63612c1 100644 --- a/fuelweb_test/helpers/ssh_manager.py +++ b/fuelweb_test/helpers/ssh_manager.py | |||
@@ -19,9 +19,10 @@ import traceback | |||
19 | from warnings import warn | 19 | from warnings import warn |
20 | 20 | ||
21 | from devops.helpers.metaclasses import SingletonMeta | 21 | from devops.helpers.metaclasses import SingletonMeta |
22 | from devops.helpers.ssh_client import SSHAuth | ||
22 | from devops.helpers.ssh_client import SSHClient | 23 | from devops.helpers.ssh_client import SSHClient |
23 | from paramiko import RSAKey | 24 | from paramiko import RSAKey |
24 | from paramiko.ssh_exception import AuthenticationException | 25 | from paramiko import SSHException |
25 | import six | 26 | import six |
26 | 27 | ||
27 | from fuelweb_test import logger | 28 | from fuelweb_test import logger |
@@ -67,8 +68,15 @@ class SSHManager(six.with_metaclass(SingletonMeta, object)): | |||
67 | self.slave_login = slave_login | 68 | self.slave_login = slave_login |
68 | self.__slave_password = slave_password | 69 | self.__slave_password = slave_password |
69 | 70 | ||
70 | @staticmethod | 71 | def _get_keys(self): |
71 | def _connect(remote): | 72 | keys = [] |
73 | admin_remote = self.get_remote(self.admin_ip) | ||
74 | key_string = '/root/.ssh/id_rsa' | ||
75 | with admin_remote.open(key_string) as f: | ||
76 | keys.append(RSAKey.from_private_key(f)) | ||
77 | return keys | ||
78 | |||
79 | def connect(self, remote): | ||
72 | """ Check if connection is stable and return this one | 80 | """ Check if connection is stable and return this one |
73 | 81 | ||
74 | :param remote: | 82 | :param remote: |
@@ -80,99 +88,129 @@ class SSHManager(six.with_metaclass(SingletonMeta, object)): | |||
80 | seconds=5, | 88 | seconds=5, |
81 | error_message="Socket timeout! Forcing reconnection"): | 89 | error_message="Socket timeout! Forcing reconnection"): |
82 | remote.check_call("cd ~") | 90 | remote.check_call("cd ~") |
83 | except: | 91 | except Exception: |
84 | logger.debug(traceback.format_exc()) | 92 | logger.debug(traceback.format_exc()) |
85 | logger.info('SSHManager: Check for current connection fails. ' | 93 | logger.debug('SSHManager: Check for current connection fails. ' |
86 | 'Trying to reconnect') | 94 | 'Trying to reconnect') |
87 | remote.reconnect() | 95 | remote = self.reconnect(remote) |
88 | return remote | 96 | return remote |
89 | 97 | ||
90 | def _get_keys(self): | 98 | def reconnect(self, remote): |
91 | keys = [] | 99 | """ Reconnect to remote or update connection |
92 | admin_remote = self.get_remote(self.admin_ip) | ||
93 | key_string = '/root/.ssh/id_rsa' | ||
94 | with admin_remote.open(key_string) as f: | ||
95 | keys.append(RSAKey.from_private_key(f)) | ||
96 | return keys | ||
97 | 100 | ||
98 | def get_remote(self, ip, port=22): | 101 | :param remote: |
99 | """ Function returns remote SSH connection to node by ip address | 102 | :return: |
103 | """ | ||
104 | ip = remote.hostname | ||
105 | port = remote.port | ||
106 | try: | ||
107 | remote.reconnect() | ||
108 | except SSHException: | ||
109 | self.update_connection(ip=ip, port=port) | ||
110 | return self.connections[(ip, port)] | ||
111 | |||
112 | def init_remote(self, ip, port=22, custom_creds=None): | ||
113 | """ Initialise connection to remote | ||
100 | 114 | ||
101 | :param ip: IP of host | 115 | :param ip: IP of host |
102 | :type ip: str | 116 | :type ip: str |
103 | :param port: port for SSH | 117 | :param port: port for SSH |
104 | :type port: int | 118 | :type port: int |
105 | :rtype: SSHClient | 119 | :param custom_creds: custom creds |
120 | :type custom_creds: dict | ||
106 | """ | 121 | """ |
107 | if (ip, port) not in self.connections: | 122 | logger.debug('SSH_MANAGER: Create new connection for ' |
108 | logger.debug('SSH_MANAGER: Create new connection for ' | 123 | '{ip}:{port}'.format(ip=ip, port=port)) |
109 | '{ip}:{port}'.format(ip=ip, port=port)) | ||
110 | 124 | ||
111 | keys = self._get_keys() if ip != self.admin_ip else [] | 125 | keys = self._get_keys() if ip != self.admin_ip else [] |
112 | if ip == self.admin_ip: | 126 | if ip == self.admin_ip: |
127 | ssh_client = SSHClient( | ||
128 | host=ip, | ||
129 | port=port, | ||
130 | auth=SSHAuth( | ||
131 | username=self.admin_login, | ||
132 | password=self.__admin_password, | ||
133 | keys=keys) | ||
134 | ) | ||
135 | ssh_client.sudo_mode = SSH_FUEL_CREDENTIALS['sudo'] | ||
136 | elif custom_creds: | ||
137 | ssh_client = SSHClient( | ||
138 | host=ip, | ||
139 | port=port, | ||
140 | auth=SSHAuth(**custom_creds)) | ||
141 | else: | ||
142 | try: | ||
113 | ssh_client = SSHClient( | 143 | ssh_client = SSHClient( |
114 | host=ip, | 144 | host=ip, |
115 | port=port, | 145 | port=port, |
116 | username=self.admin_login, | 146 | auth=SSHAuth( |
117 | password=self.__admin_password, | ||
118 | private_keys=keys | ||
119 | ) | ||
120 | ssh_client.sudo_mode = SSH_FUEL_CREDENTIALS['sudo'] | ||
121 | else: | ||
122 | try: | ||
123 | ssh_client = SSHClient( | ||
124 | host=ip, | ||
125 | port=port, | ||
126 | username=self.slave_login, | 147 | username=self.slave_login, |
127 | password=self.__slave_password, | 148 | password=self.__slave_password, |
128 | private_keys=keys | 149 | keys=keys) |
129 | ) | 150 | ) |
130 | except AuthenticationException: | 151 | except SSHException: |
131 | ssh_client = SSHClient( | 152 | ssh_client = SSHClient( |
132 | host=ip, | 153 | host=ip, |
133 | port=port, | 154 | port=port, |
155 | auth=SSHAuth( | ||
134 | username=self.slave_fallback_login, | 156 | username=self.slave_fallback_login, |
135 | password=self.__slave_password, | 157 | password=self.__slave_password, |
136 | private_keys=keys | 158 | keys=keys) |
137 | ) | 159 | ) |
138 | ssh_client.sudo_mode = SSH_SLAVE_CREDENTIALS['sudo'] | 160 | ssh_client.sudo_mode = SSH_SLAVE_CREDENTIALS['sudo'] |
139 | self.connections[(ip, port)] = ssh_client | 161 | |
140 | logger.debug('SSH_MANAGER: Return existed connection for ' | 162 | self.connections[(ip, port)] = ssh_client |
141 | '{ip}:{port}'.format(ip=ip, port=port)) | 163 | logger.debug('SSH_MANAGER: New connection for ' |
164 | '{ip}:{port} is created'.format(ip=ip, port=port)) | ||
165 | |||
166 | def get_remote(self, ip, port=22): | ||
167 | """ Function returns remote SSH connection to node by ip address | ||
168 | |||
169 | :param ip: IP of host | ||
170 | :type ip: str | ||
171 | :param port: port for SSH | ||
172 | :type port: int | ||
173 | :rtype: SSHClient | ||
174 | """ | ||
175 | if (ip, port) in self.connections: | ||
176 | logger.debug('SSH_MANAGER: Return existed connection for ' | ||
177 | '{ip}:{port}'.format(ip=ip, port=port)) | ||
178 | else: | ||
179 | self.init_remote(ip=ip, port=port) | ||
142 | logger.debug('SSH_MANAGER: Connections {0}'.format(self.connections)) | 180 | logger.debug('SSH_MANAGER: Connections {0}'.format(self.connections)) |
143 | return self._connect(self.connections[(ip, port)]) | 181 | return self.connect(self.connections[(ip, port)]) |
144 | 182 | ||
145 | def update_connection(self, ip, login=None, password=None, | 183 | def update_connection(self, ip, port=22, login=None, password=None, |
146 | keys=None, port=22): | 184 | keys=None): |
147 | """Update existed connection | 185 | """Update existed connection |
148 | 186 | ||
149 | :param ip: host ip string | 187 | :param ip: host ip string |
188 | :param port: ssh port int | ||
150 | :param login: login string | 189 | :param login: login string |
151 | :param password: password string | 190 | :param password: password string |
152 | :param keys: list of keys | 191 | :param keys: list of keys |
153 | :param port: ssh port int | ||
154 | :return: None | 192 | :return: None |
155 | """ | 193 | """ |
156 | if (ip, port) in self.connections: | 194 | if (ip, port) in self.connections: |
157 | logger.info('SSH_MANAGER: Close connection for {ip}:{port}'.format( | 195 | logger.debug('SSH_MANAGER: Close connection for {ip}:{port}' |
158 | ip=ip, port=port)) | 196 | .format(ip=ip, port=port)) |
159 | self.connections[(ip, port)].clear() | 197 | ssh_client = self.connections.pop((ip, port)) |
160 | logger.info('SSH_MANAGER: Create new connection for ' | 198 | ssh_client.close() |
161 | '{ip}:{port}'.format(ip=ip, port=port)) | 199 | if login and (password or keys): |
162 | 200 | custom_creds = { | |
163 | self.connections[(ip, port)] = SSHClient( | 201 | 'username': login, |
164 | host=ip, | 202 | 'password': password, |
165 | port=port, | 203 | 'keys': keys |
166 | username=login, | 204 | } |
167 | password=password, | 205 | else: |
168 | private_keys=keys if keys is not None else [] | 206 | custom_creds = None |
169 | ) | 207 | self.init_remote(ip=ip, port=port, custom_creds=custom_creds) |
170 | 208 | ||
171 | def clean_all_connections(self): | 209 | def clean_all_connections(self): |
172 | for (ip, port), connection in self.connections.items(): | 210 | for (ip, port), connection in self.connections.items(): |
173 | connection.clear() | 211 | connection.clear() |
174 | logger.info('SSH_MANAGER: Close connection for {ip}:{port}'.format( | 212 | logger.debug('SSH_MANAGER: Close connection for {ip}:{port}' |
175 | ip=ip, port=port)) | 213 | .format(ip=ip, port=port)) |
176 | 214 | ||
177 | def execute(self, ip, cmd, port=22, sudo=None): | 215 | def execute(self, ip, cmd, port=22, sudo=None): |
178 | remote = self.get_remote(ip=ip, port=port) | 216 | remote = self.get_remote(ip=ip, port=port) |
diff --git a/fuelweb_test/models/environment.py b/fuelweb_test/models/environment.py index 08dd32f..3eeb915 100644 --- a/fuelweb_test/models/environment.py +++ b/fuelweb_test/models/environment.py | |||
@@ -377,6 +377,7 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)): | |||
377 | ) | 377 | ) |
378 | self.ssh_manager.update_connection( | 378 | self.ssh_manager.update_connection( |
379 | ip=self.ssh_manager.admin_ip, | 379 | ip=self.ssh_manager.admin_ip, |
380 | port=22, | ||
380 | login=new_login, | 381 | login=new_login, |
381 | password=new_password | 382 | password=new_password |
382 | ) | 383 | ) |
diff --git a/fuelweb_test/tests/test_multipath_devices.py b/fuelweb_test/tests/test_multipath_devices.py index 0d07aac..3fc927f 100644 --- a/fuelweb_test/tests/test_multipath_devices.py +++ b/fuelweb_test/tests/test_multipath_devices.py | |||
@@ -25,7 +25,6 @@ from fuelweb_test.settings import MULTIPATH | |||
25 | from fuelweb_test.settings import MULTIPATH_TEMPLATE | 25 | from fuelweb_test.settings import MULTIPATH_TEMPLATE |
26 | from fuelweb_test.settings import NEUTRON_SEGMENT | 26 | from fuelweb_test.settings import NEUTRON_SEGMENT |
27 | from fuelweb_test.settings import SLAVE_MULTIPATH_DISKS_COUNT | 27 | from fuelweb_test.settings import SLAVE_MULTIPATH_DISKS_COUNT |
28 | from fuelweb_test.settings import SSH_FUEL_CREDENTIALS | ||
29 | from fuelweb_test.settings import REPLACE_DEFAULT_REPOS | 28 | from fuelweb_test.settings import REPLACE_DEFAULT_REPOS |
30 | from fuelweb_test.settings import REPLACE_DEFAULT_REPOS_ONLY_ONCE | 29 | from fuelweb_test.settings import REPLACE_DEFAULT_REPOS_ONLY_ONCE |
31 | from fuelweb_test.tests import base_test_case | 30 | from fuelweb_test.tests import base_test_case |
@@ -53,9 +52,7 @@ class TestMultipath(base_test_case.TestBasic): | |||
53 | """ | 52 | """ |
54 | cmd = "multipath -l -v2" | 53 | cmd = "multipath -l -v2" |
55 | 54 | ||
56 | ssh_manager.update_connection(ip, SSH_FUEL_CREDENTIALS['login'], | 55 | ssh_manager.update_connection(ip) |
57 | SSH_FUEL_CREDENTIALS['password'], | ||
58 | keys=ssh_manager._get_keys()) | ||
59 | ssh_manager.get_remote(ip) | 56 | ssh_manager.get_remote(ip) |
60 | result = ssh_manager.execute_on_remote( | 57 | result = ssh_manager.execute_on_remote( |
61 | ip=ip, | 58 | ip=ip, |
@@ -106,9 +103,7 @@ class TestMultipath(base_test_case.TestBasic): | |||
106 | """ | 103 | """ |
107 | cmd = "lsblk -lo NAME,TYPE,MOUNTPOINT | grep '/$' | grep -c lvm" | 104 | cmd = "lsblk -lo NAME,TYPE,MOUNTPOINT | grep '/$' | grep -c lvm" |
108 | 105 | ||
109 | ssh_manager.update_connection(ip, SSH_FUEL_CREDENTIALS['login'], | 106 | ssh_manager.update_connection(ip) |
110 | SSH_FUEL_CREDENTIALS['password'], | ||
111 | keys=ssh_manager._get_keys()) | ||
112 | ssh_manager.get_remote(ip) | 107 | ssh_manager.get_remote(ip) |
113 | result = ssh_manager.execute_on_remote( | 108 | result = ssh_manager.execute_on_remote( |
114 | ip=ip, | 109 | ip=ip, |