summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTom Barron <tpb@dyncloud.net>2018-12-29 20:41:14 -0600
committerTom Barron <tpb@dyncloud.net>2018-12-30 11:04:16 -0600
commit7548706b0928f2c23bf27e10d116ceca585660e4 (patch)
tree9ba5778ad99c38f8e7a5c580c635c401e856fc92
parentcf41d1867c35ccb116da94a559fd30efc00b1548 (diff)
Adjust ssh timeouts
Generic driver jobs are failing because of timeouts when establishing the initial ssh connection from manila-share to the service VM. Bump up the default value of the connection timeout for paramiko client and also set the banner timeout since the failure occurred during banner exchange. Set the two timeouts to the same value for now. This ensures that the connection timeout is at least as long as the banner timeout and there is no current need in manila to control these independently. This is more of a workaround than a real fix since a real fix would remove the delay during banner exchange. I suspect that the real fix will need to be in neutron/ovs though. Change-Id: Ib5e59faaf9667b9cb5e7d4072531b7d6c3d4da39 Partial-bug: #1807216
Notes
Notes (review): Code-Review+2: Rodrigo Barbieri <rodrigo.barbieri2010@gmail.com> Code-Review+2: Thomas Bechtold <tbechtold@suse.com> Workflow+1: Rodrigo Barbieri <rodrigo.barbieri2010@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Wed, 02 Jan 2019 12:12:35 +0000 Reviewed-on: https://review.openstack.org/627794 Project: openstack/manila Branch: refs/heads/master
-rwxr-xr-xdevstack/plugin.sh4
-rw-r--r--devstack/settings3
-rw-r--r--manila/tests/share/drivers/hitachi/hnas/test_ssh.py3
-rw-r--r--manila/tests/test_utils.py11
-rw-r--r--manila/utils.py14
5 files changed, 28 insertions, 7 deletions
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index c78756e..696fdea 100755
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -102,6 +102,10 @@ function configure_default_backends {
102 if [ $(trueorfalse False MANILA_USE_SERVICE_INSTANCE_PASSWORD) == True ]; then 102 if [ $(trueorfalse False MANILA_USE_SERVICE_INSTANCE_PASSWORD) == True ]; then
103 iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD 103 iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD
104 fi 104 fi
105
106 if [ "$SHARE_DRIVER" == "manila.share.drivers.generic.GenericShareDriver" ]; then
107 iniset $MANILA_CONF $group_name ssh_conn_timeout $MANILA_SSH_TIMEOUT
108 fi
105 done 109 done
106} 110}
107 111
diff --git a/devstack/settings b/devstack/settings
index 2d7bff4..d41cbeb 100644
--- a/devstack/settings
+++ b/devstack/settings
@@ -187,6 +187,9 @@ MANILA_DOCKER_IMAGE_URL=${MANILA_DOCKER_IMAGE_URL:-"https://github.com/a-ovchinn
187MANILA_NETWORK_API_CLASS=${MANILA_NETWORK_API_CLASS:-"manila.network.neutron.neutron_network_plugin.NeutronBindNetworkPlugin"} 187MANILA_NETWORK_API_CLASS=${MANILA_NETWORK_API_CLASS:-"manila.network.neutron.neutron_network_plugin.NeutronBindNetworkPlugin"}
188MANILA_NEUTRON_VNIC_TYPE=${MANILA_NEUTRON_VNIC_TYPE:-"normal"} 188MANILA_NEUTRON_VNIC_TYPE=${MANILA_NEUTRON_VNIC_TYPE:-"normal"}
189 189
190# SSH TIMEOUT
191MANILA_SSH_TIMEOUT=${MANILA_SSH_TIMEOUT:-180}
192
190# Admin Network setup 193# Admin Network setup
191MANILA_ADMIN_NET_RANGE=${MANILA_ADMIN_NET_RANGE:=10.2.5.0/24} 194MANILA_ADMIN_NET_RANGE=${MANILA_ADMIN_NET_RANGE:=10.2.5.0/24}
192 195
diff --git a/manila/tests/share/drivers/hitachi/hnas/test_ssh.py b/manila/tests/share/drivers/hitachi/hnas/test_ssh.py
index 6261d41..6889dc0 100644
--- a/manila/tests/share/drivers/hitachi/hnas/test_ssh.py
+++ b/manila/tests/share/drivers/hitachi/hnas/test_ssh.py
@@ -1491,7 +1491,8 @@ class HNASSSHTestCase(test.TestCase):
1491 look_for_keys=False, 1491 look_for_keys=False,
1492 timeout=None, 1492 timeout=None,
1493 password=self.password, 1493 password=self.password,
1494 port=self.port) 1494 port=self.port,
1495 banner_timeout=None)
1495 self.assertIn('Request submitted successfully.', output) 1496 self.assertIn('Request submitted successfully.', output)
1496 1497
1497 def test__execute_ssh_exception(self): 1498 def test__execute_ssh_exception(self):
diff --git a/manila/tests/test_utils.py b/manila/tests/test_utils.py
index b076933..530fb26 100644
--- a/manila/tests/test_utils.py
+++ b/manila/tests/test_utils.py
@@ -225,7 +225,8 @@ class FakeSSHClient(object):
225 pass 225 pass
226 226
227 def connect(self, ip, port=22, username=None, password=None, 227 def connect(self, ip, port=22, username=None, password=None,
228 key_filename=None, look_for_keys=None, timeout=10): 228 key_filename=None, look_for_keys=None, timeout=10,
229 banner_timeout=10):
229 pass 230 pass
230 231
231 def get_transport(self): 232 def get_transport(self):
@@ -284,7 +285,7 @@ class SSHPoolTestCase(test.TestCase):
284 fake_ssh_client.connect.assert_called_once_with( 285 fake_ssh_client.connect.assert_called_once_with(
285 "127.0.0.1", port=22, username="test", 286 "127.0.0.1", port=22, username="test",
286 password="test", key_filename=None, look_for_keys=False, 287 password="test", key_filename=None, look_for_keys=False,
287 timeout=10) 288 timeout=10, banner_timeout=10)
288 289
289 def test_create_ssh_with_key(self): 290 def test_create_ssh_with_key(self):
290 path_to_private_key = "/fakepath/to/privatekey" 291 path_to_private_key = "/fakepath/to/privatekey"
@@ -297,7 +298,7 @@ class SSHPoolTestCase(test.TestCase):
297 fake_ssh_client.connect.assert_called_once_with( 298 fake_ssh_client.connect.assert_called_once_with(
298 "127.0.0.1", port=22, username="test", password=None, 299 "127.0.0.1", port=22, username="test", password=None,
299 key_filename=path_to_private_key, look_for_keys=False, 300 key_filename=path_to_private_key, look_for_keys=False,
300 timeout=10) 301 timeout=10, banner_timeout=10)
301 302
302 def test_create_ssh_with_nothing(self): 303 def test_create_ssh_with_nothing(self):
303 fake_ssh_client = mock.Mock() 304 fake_ssh_client = mock.Mock()
@@ -308,7 +309,7 @@ class SSHPoolTestCase(test.TestCase):
308 fake_ssh_client.connect.assert_called_once_with( 309 fake_ssh_client.connect.assert_called_once_with(
309 "127.0.0.1", port=22, username="test", password=None, 310 "127.0.0.1", port=22, username="test", password=None,
310 key_filename=None, look_for_keys=True, 311 key_filename=None, look_for_keys=True,
311 timeout=10) 312 timeout=10, banner_timeout=10)
312 313
313 def test_create_ssh_error_connecting(self): 314 def test_create_ssh_error_connecting(self):
314 attrs = {'connect.side_effect': paramiko.SSHException, } 315 attrs = {'connect.side_effect': paramiko.SSHException, }
@@ -320,7 +321,7 @@ class SSHPoolTestCase(test.TestCase):
320 fake_ssh_client.connect.assert_called_once_with( 321 fake_ssh_client.connect.assert_called_once_with(
321 "127.0.0.1", port=22, username="test", password=None, 322 "127.0.0.1", port=22, username="test", password=None,
322 key_filename=None, look_for_keys=True, 323 key_filename=None, look_for_keys=True,
323 timeout=10) 324 timeout=10, banner_timeout=10)
324 325
325 def test_closed_reopend_ssh_connections(self): 326 def test_closed_reopend_ssh_connections(self):
326 with mock.patch.object(paramiko, "SSHClient", 327 with mock.patch.object(paramiko, "SSHClient",
diff --git a/manila/utils.py b/manila/utils.py
index a55d1b6..bb417ec 100644
--- a/manila/utils.py
+++ b/manila/utils.py
@@ -128,13 +128,25 @@ class SSHPool(pools.Pool):
128 elif self.password: 128 elif self.password:
129 look_for_keys = False 129 look_for_keys = False
130 try: 130 try:
131 LOG.debug("ssh.connect: ip: %s, port: %s, username: %s, "
132 "password: %s, key_filename: %s, look_for_keys: %s, "
133 "timeout: %s, banner_timeout: %s",
134 self.ip,
135 self.port,
136 self.login,
137 self.password,
138 self.path_to_private_key,
139 look_for_keys,
140 self.conn_timeout,
141 self.conn_timeout)
131 ssh.connect(self.ip, 142 ssh.connect(self.ip,
132 port=self.port, 143 port=self.port,
133 username=self.login, 144 username=self.login,
134 password=self.password, 145 password=self.password,
135 key_filename=self.path_to_private_key, 146 key_filename=self.path_to_private_key,
136 look_for_keys=look_for_keys, 147 look_for_keys=look_for_keys,
137 timeout=self.conn_timeout) 148 timeout=self.conn_timeout,
149 banner_timeout=self.conn_timeout)
138 # Paramiko by default sets the socket timeout to 0.1 seconds, 150 # Paramiko by default sets the socket timeout to 0.1 seconds,
139 # ignoring what we set through the sshclient. This doesn't help for 151 # ignoring what we set through the sshclient. This doesn't help for
140 # keeping long lived connections. Hence we have to bypass it, by 152 # keeping long lived connections. Hence we have to bypass it, by