From 2793bd5e8315e807e2395e53c5fbfc6f205cca8b Mon Sep 17 00:00:00 2001 From: Tom Barron Date: Sat, 29 Dec 2018 20:41:14 -0600 Subject: [PATCH] Adjust ssh timeouts Generic driver jobs are failing because of timeouts when establishing the initial ssh connection from manila-share to the service VM. Bump up the default value of the connection timeout for paramiko client and also set the banner timeout since the failure occurred during banner exchange. Set the two timeouts to the same value for now. This ensures that the connection timeout is at least as long as the banner timeout and there is no current need in manila to control these independently. This is more of a workaround than a real fix since a real fix would remove the delay during banner exchange. I suspect that the real fix will need to be in neutron/ovs though. Change-Id: Ib5e59faaf9667b9cb5e7d4072531b7d6c3d4da39 Partial-bug: #1807216 (cherry picked from commit 7548706b0928f2c23bf27e10d116ceca585660e4) (cherry picked from commit 8919db0186f8da9f2f492d80c8a3700c3d9367d1) (cherry picked from commit 2abde73d1e4a91569b2c844632acc10676b11e26) --- devstack/plugin.sh | 4 ++++ devstack/settings | 3 +++ .../tests/share/drivers/hitachi/hnas/test_ssh.py | 3 ++- manila/tests/test_utils.py | 11 ++++++----- manila/utils.py | 14 +++++++++++++- 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 07b7681182..2ee6d144a1 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -100,6 +100,10 @@ function configure_default_backends { if [ $(trueorfalse False MANILA_USE_SERVICE_INSTANCE_PASSWORD) == True ]; then iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD fi + + if [ "$SHARE_DRIVER" == "manila.share.drivers.generic.GenericShareDriver" ]; then + iniset $MANILA_CONF $group_name ssh_conn_timeout $MANILA_SSH_TIMEOUT + fi done } diff --git a/devstack/settings b/devstack/settings index 8e06673bb6..0f3b74a7e2 100644 --- a/devstack/settings +++ b/devstack/settings @@ -183,6 +183,9 @@ MANILA_DOCKER_IMAGE_URL=${MANILA_DOCKER_IMAGE_URL:-"https://github.com/a-ovchinn MANILA_NETWORK_API_CLASS=${MANILA_NETWORK_API_CLASS:-"manila.network.neutron.neutron_network_plugin.NeutronBindNetworkPlugin"} MANILA_NEUTRON_VNIC_TYPE=${MANILA_NEUTRON_VNIC_TYPE:-"normal"} +# SSH TIMEOUT +MANILA_SSH_TIMEOUT=${MANILA_SSH_TIMEOUT:-180} + # Admin Network setup MANILA_ADMIN_NET_RANGE=${MANILA_ADMIN_NET_RANGE:=10.2.5.0/24} diff --git a/manila/tests/share/drivers/hitachi/hnas/test_ssh.py b/manila/tests/share/drivers/hitachi/hnas/test_ssh.py index 6261d418d7..6889dc06ff 100644 --- a/manila/tests/share/drivers/hitachi/hnas/test_ssh.py +++ b/manila/tests/share/drivers/hitachi/hnas/test_ssh.py @@ -1491,7 +1491,8 @@ class HNASSSHTestCase(test.TestCase): look_for_keys=False, timeout=None, password=self.password, - port=self.port) + port=self.port, + banner_timeout=None) self.assertIn('Request submitted successfully.', output) def test__execute_ssh_exception(self): diff --git a/manila/tests/test_utils.py b/manila/tests/test_utils.py index 99fc6cb4e3..38963946f2 100644 --- a/manila/tests/test_utils.py +++ b/manila/tests/test_utils.py @@ -225,7 +225,8 @@ class FakeSSHClient(object): pass def connect(self, ip, port=22, username=None, password=None, - key_filename=None, look_for_keys=None, timeout=10): + key_filename=None, look_for_keys=None, timeout=10, + banner_timeout=10): pass def get_transport(self): @@ -284,7 +285,7 @@ class SSHPoolTestCase(test.TestCase): fake_ssh_client.connect.assert_called_once_with( "127.0.0.1", port=22, username="test", password="test", key_filename=None, look_for_keys=False, - timeout=10) + timeout=10, banner_timeout=10) def test_create_ssh_with_key(self): path_to_private_key = "/fakepath/to/privatekey" @@ -297,7 +298,7 @@ class SSHPoolTestCase(test.TestCase): fake_ssh_client.connect.assert_called_once_with( "127.0.0.1", port=22, username="test", password=None, key_filename=path_to_private_key, look_for_keys=False, - timeout=10) + timeout=10, banner_timeout=10) def test_create_ssh_with_nothing(self): fake_ssh_client = mock.Mock() @@ -308,7 +309,7 @@ class SSHPoolTestCase(test.TestCase): fake_ssh_client.connect.assert_called_once_with( "127.0.0.1", port=22, username="test", password=None, key_filename=None, look_for_keys=True, - timeout=10) + timeout=10, banner_timeout=10) def test_create_ssh_error_connecting(self): attrs = {'connect.side_effect': paramiko.SSHException, } @@ -320,7 +321,7 @@ class SSHPoolTestCase(test.TestCase): fake_ssh_client.connect.assert_called_once_with( "127.0.0.1", port=22, username="test", password=None, key_filename=None, look_for_keys=True, - timeout=10) + timeout=10, banner_timeout=10) def test_closed_reopend_ssh_connections(self): with mock.patch.object(paramiko, "SSHClient", diff --git a/manila/utils.py b/manila/utils.py index 2cf94fc22f..8a8edb3b6d 100644 --- a/manila/utils.py +++ b/manila/utils.py @@ -127,13 +127,25 @@ class SSHPool(pools.Pool): elif self.password: look_for_keys = False try: + LOG.debug("ssh.connect: ip: %s, port: %s, username: %s, " + "password: %s, key_filename: %s, look_for_keys: %s, " + "timeout: %s, banner_timeout: %s", + self.ip, + self.port, + self.login, + self.password, + self.path_to_private_key, + look_for_keys, + self.conn_timeout, + self.conn_timeout) ssh.connect(self.ip, port=self.port, username=self.login, password=self.password, key_filename=self.path_to_private_key, look_for_keys=look_for_keys, - timeout=self.conn_timeout) + timeout=self.conn_timeout, + banner_timeout=self.conn_timeout) # Paramiko by default sets the socket timeout to 0.1 seconds, # ignoring what we set through the sshclient. This doesn't help for # keeping long lived connections. Hence we have to bypass it, by