summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTom Barron <tpb@dyncloud.net>2018-12-29 20:41:14 -0600
committerTom Barron <tpb@dyncloud.net>2019-01-16 21:48:46 +0000
commit0f061ac34b8ce63de7e6dd6bd617ddb70b88cc46 (patch)
treeb62140e087984f66288c06c00ec15429eef3f45f
parent895bfd4f7afc568b14d95b9c9bcc5e21d09b1b02 (diff)
Adjust ssh timeoutsstable/ocata
Generic driver jobs are failing because of timeouts when establishing the initial ssh connection from manila-share to the service VM. Bump up the default value of the connection timeout for paramiko client and also set the banner timeout since the failure occurred during banner exchange. Set the two timeouts to the same value for now. This ensures that the connection timeout is at least as long as the banner timeout and there is no current need in manila to control these independently. This is more of a workaround than a real fix since a real fix would remove the delay during banner exchange. I suspect that the real fix will need to be in neutron/ovs though. Change-Id: Ib5e59faaf9667b9cb5e7d4072531b7d6c3d4da39 Partial-bug: #1807216 (cherry picked from commit 7548706b0928f2c23bf27e10d116ceca585660e4) (cherry picked from commit 8919db0186f8da9f2f492d80c8a3700c3d9367d1) (cherry picked from commit 2abde73d1e4a91569b2c844632acc10676b11e26) (cherry picked from commit 2793bd5e8315e807e2395e53c5fbfc6f205cca8b)
Notes
Notes (review): Code-Review+2: Thomas Bechtold <tbechtold@suse.com> Code-Review+2: Goutham Pacha Ravi <gouthampravi@gmail.com> Workflow+1: Goutham Pacha Ravi <gouthampravi@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Thu, 24 Jan 2019 00:52:47 +0000 Reviewed-on: https://review.openstack.org/630965 Project: openstack/manila Branch: refs/heads/stable/ocata
-rwxr-xr-xdevstack/plugin.sh4
-rw-r--r--devstack/settings3
-rw-r--r--manila/tests/share/drivers/hitachi/hnas/test_ssh.py3
-rw-r--r--manila/tests/test_utils.py11
-rw-r--r--manila/utils.py14
5 files changed, 28 insertions, 7 deletions
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index c89f84c..40615cc 100755
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -84,6 +84,10 @@ function configure_default_backends {
84 if [ $(trueorfalse False MANILA_USE_SERVICE_INSTANCE_PASSWORD) == True ]; then 84 if [ $(trueorfalse False MANILA_USE_SERVICE_INSTANCE_PASSWORD) == True ]; then
85 iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD 85 iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD
86 fi 86 fi
87
88 if [ "$SHARE_DRIVER" == "manila.share.drivers.generic.GenericShareDriver" ]; then
89 iniset $MANILA_CONF $group_name ssh_conn_timeout $MANILA_SSH_TIMEOUT
90 fi
87 done 91 done
88} 92}
89 93
diff --git a/devstack/settings b/devstack/settings
index cc3c68c..d0c5f76 100644
--- a/devstack/settings
+++ b/devstack/settings
@@ -177,6 +177,9 @@ MANILA_DOCKER_IMAGE_URL=${MANILA_DOCKER_IMAGE_URL:-"https://github.com/a-ovchinn
177MANILA_NETWORK_API_CLASS=${MANILA_NETWORK_API_CLASS:-"manila.network.neutron.neutron_network_plugin.NeutronBindNetworkPlugin"} 177MANILA_NETWORK_API_CLASS=${MANILA_NETWORK_API_CLASS:-"manila.network.neutron.neutron_network_plugin.NeutronBindNetworkPlugin"}
178MANILA_NEUTRON_VNIC_TYPE=${MANILA_NEUTRON_VNIC_TYPE:-"normal"} 178MANILA_NEUTRON_VNIC_TYPE=${MANILA_NEUTRON_VNIC_TYPE:-"normal"}
179 179
180# SSH TIMEOUT
181MANILA_SSH_TIMEOUT=${MANILA_SSH_TIMEOUT:-180}
182
180# Admin Network setup 183# Admin Network setup
181MANILA_ADMIN_NET_RANGE=${MANILA_ADMIN_NET_RANGE:=10.2.5.0/24} 184MANILA_ADMIN_NET_RANGE=${MANILA_ADMIN_NET_RANGE:=10.2.5.0/24}
182 185
diff --git a/manila/tests/share/drivers/hitachi/hnas/test_ssh.py b/manila/tests/share/drivers/hitachi/hnas/test_ssh.py
index 4526267..10626bd 100644
--- a/manila/tests/share/drivers/hitachi/hnas/test_ssh.py
+++ b/manila/tests/share/drivers/hitachi/hnas/test_ssh.py
@@ -1381,7 +1381,8 @@ class HNASSSHTestCase(test.TestCase):
1381 look_for_keys=False, 1381 look_for_keys=False,
1382 timeout=None, 1382 timeout=None,
1383 password=self.password, 1383 password=self.password,
1384 port=self.port) 1384 port=self.port,
1385 banner_timeout=None)
1385 self.assertIn('Request submitted successfully.', output) 1386 self.assertIn('Request submitted successfully.', output)
1386 1387
1387 def test__execute_ssh_exception(self): 1388 def test__execute_ssh_exception(self):
diff --git a/manila/tests/test_utils.py b/manila/tests/test_utils.py
index c8bc591..039057b 100644
--- a/manila/tests/test_utils.py
+++ b/manila/tests/test_utils.py
@@ -223,7 +223,8 @@ class FakeSSHClient(object):
223 pass 223 pass
224 224
225 def connect(self, ip, port=22, username=None, password=None, 225 def connect(self, ip, port=22, username=None, password=None,
226 key_filename=None, look_for_keys=None, timeout=10): 226 key_filename=None, look_for_keys=None, timeout=10,
227 banner_timeout=10):
227 pass 228 pass
228 229
229 def get_transport(self): 230 def get_transport(self):
@@ -282,7 +283,7 @@ class SSHPoolTestCase(test.TestCase):
282 fake_ssh_client.connect.assert_called_once_with( 283 fake_ssh_client.connect.assert_called_once_with(
283 "127.0.0.1", port=22, username="test", 284 "127.0.0.1", port=22, username="test",
284 password="test", key_filename=None, look_for_keys=False, 285 password="test", key_filename=None, look_for_keys=False,
285 timeout=10) 286 timeout=10, banner_timeout=10)
286 287
287 def test_create_ssh_with_key(self): 288 def test_create_ssh_with_key(self):
288 path_to_private_key = "/fakepath/to/privatekey" 289 path_to_private_key = "/fakepath/to/privatekey"
@@ -295,7 +296,7 @@ class SSHPoolTestCase(test.TestCase):
295 fake_ssh_client.connect.assert_called_once_with( 296 fake_ssh_client.connect.assert_called_once_with(
296 "127.0.0.1", port=22, username="test", password=None, 297 "127.0.0.1", port=22, username="test", password=None,
297 key_filename=path_to_private_key, look_for_keys=False, 298 key_filename=path_to_private_key, look_for_keys=False,
298 timeout=10) 299 timeout=10, banner_timeout=10)
299 300
300 def test_create_ssh_with_nothing(self): 301 def test_create_ssh_with_nothing(self):
301 fake_ssh_client = mock.Mock() 302 fake_ssh_client = mock.Mock()
@@ -306,7 +307,7 @@ class SSHPoolTestCase(test.TestCase):
306 fake_ssh_client.connect.assert_called_once_with( 307 fake_ssh_client.connect.assert_called_once_with(
307 "127.0.0.1", port=22, username="test", password=None, 308 "127.0.0.1", port=22, username="test", password=None,
308 key_filename=None, look_for_keys=True, 309 key_filename=None, look_for_keys=True,
309 timeout=10) 310 timeout=10, banner_timeout=10)
310 311
311 def test_create_ssh_error_connecting(self): 312 def test_create_ssh_error_connecting(self):
312 attrs = {'connect.side_effect': paramiko.SSHException, } 313 attrs = {'connect.side_effect': paramiko.SSHException, }
@@ -318,7 +319,7 @@ class SSHPoolTestCase(test.TestCase):
318 fake_ssh_client.connect.assert_called_once_with( 319 fake_ssh_client.connect.assert_called_once_with(
319 "127.0.0.1", port=22, username="test", password=None, 320 "127.0.0.1", port=22, username="test", password=None,
320 key_filename=None, look_for_keys=True, 321 key_filename=None, look_for_keys=True,
321 timeout=10) 322 timeout=10, banner_timeout=10)
322 323
323 def test_closed_reopend_ssh_connections(self): 324 def test_closed_reopend_ssh_connections(self):
324 with mock.patch.object(paramiko, "SSHClient", 325 with mock.patch.object(paramiko, "SSHClient",
diff --git a/manila/utils.py b/manila/utils.py
index 5cc2282..13218b8 100644
--- a/manila/utils.py
+++ b/manila/utils.py
@@ -103,13 +103,25 @@ class SSHPool(pools.Pool):
103 elif self.password: 103 elif self.password:
104 look_for_keys = False 104 look_for_keys = False
105 try: 105 try:
106 LOG.debug("ssh.connect: ip: %s, port: %s, username: %s, "
107 "password: %s, key_filename: %s, look_for_keys: %s, "
108 "timeout: %s, banner_timeout: %s",
109 self.ip,
110 self.port,
111 self.login,
112 self.password,
113 self.path_to_private_key,
114 look_for_keys,
115 self.conn_timeout,
116 self.conn_timeout)
106 ssh.connect(self.ip, 117 ssh.connect(self.ip,
107 port=self.port, 118 port=self.port,
108 username=self.login, 119 username=self.login,
109 password=self.password, 120 password=self.password,
110 key_filename=self.path_to_private_key, 121 key_filename=self.path_to_private_key,
111 look_for_keys=look_for_keys, 122 look_for_keys=look_for_keys,
112 timeout=self.conn_timeout) 123 timeout=self.conn_timeout,
124 banner_timeout=self.conn_timeout)
113 if self.conn_timeout: 125 if self.conn_timeout:
114 transport = ssh.get_transport() 126 transport = ssh.get_transport()
115 transport.set_keepalive(self.conn_timeout) 127 transport.set_keepalive(self.conn_timeout)