From 0f061ac34b8ce63de7e6dd6bd617ddb70b88cc46 Mon Sep 17 00:00:00 2001 From: Tom Barron Date: Sat, 29 Dec 2018 20:41:14 -0600 Subject: [PATCH] Adjust ssh timeouts Generic driver jobs are failing because of timeouts when establishing the initial ssh connection from manila-share to the service VM. Bump up the default value of the connection timeout for paramiko client and also set the banner timeout since the failure occurred during banner exchange. Set the two timeouts to the same value for now. This ensures that the connection timeout is at least as long as the banner timeout and there is no current need in manila to control these independently. This is more of a workaround than a real fix since a real fix would remove the delay during banner exchange. I suspect that the real fix will need to be in neutron/ovs though. Change-Id: Ib5e59faaf9667b9cb5e7d4072531b7d6c3d4da39 Partial-bug: #1807216 (cherry picked from commit 7548706b0928f2c23bf27e10d116ceca585660e4) (cherry picked from commit 8919db0186f8da9f2f492d80c8a3700c3d9367d1) (cherry picked from commit 2abde73d1e4a91569b2c844632acc10676b11e26) (cherry picked from commit 2793bd5e8315e807e2395e53c5fbfc6f205cca8b) --- devstack/plugin.sh | 4 ++++ devstack/settings | 3 +++ .../tests/share/drivers/hitachi/hnas/test_ssh.py | 3 ++- manila/tests/test_utils.py | 11 ++++++----- manila/utils.py | 14 +++++++++++++- 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/devstack/plugin.sh b/devstack/plugin.sh index c89f84cea7..40615cc9d6 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -84,6 +84,10 @@ function configure_default_backends { if [ $(trueorfalse False MANILA_USE_SERVICE_INSTANCE_PASSWORD) == True ]; then iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD fi + + if [ "$SHARE_DRIVER" == "manila.share.drivers.generic.GenericShareDriver" ]; then + iniset $MANILA_CONF $group_name ssh_conn_timeout $MANILA_SSH_TIMEOUT + fi done } diff --git a/devstack/settings b/devstack/settings index cc3c68c369..d0c5f76d94 100644 --- a/devstack/settings +++ b/devstack/settings @@ -177,6 +177,9 @@ MANILA_DOCKER_IMAGE_URL=${MANILA_DOCKER_IMAGE_URL:-"https://github.com/a-ovchinn MANILA_NETWORK_API_CLASS=${MANILA_NETWORK_API_CLASS:-"manila.network.neutron.neutron_network_plugin.NeutronBindNetworkPlugin"} MANILA_NEUTRON_VNIC_TYPE=${MANILA_NEUTRON_VNIC_TYPE:-"normal"} +# SSH TIMEOUT +MANILA_SSH_TIMEOUT=${MANILA_SSH_TIMEOUT:-180} + # Admin Network setup MANILA_ADMIN_NET_RANGE=${MANILA_ADMIN_NET_RANGE:=10.2.5.0/24} diff --git a/manila/tests/share/drivers/hitachi/hnas/test_ssh.py b/manila/tests/share/drivers/hitachi/hnas/test_ssh.py index 4526267d29..10626bdc62 100644 --- a/manila/tests/share/drivers/hitachi/hnas/test_ssh.py +++ b/manila/tests/share/drivers/hitachi/hnas/test_ssh.py @@ -1381,7 +1381,8 @@ class HNASSSHTestCase(test.TestCase): look_for_keys=False, timeout=None, password=self.password, - port=self.port) + port=self.port, + banner_timeout=None) self.assertIn('Request submitted successfully.', output) def test__execute_ssh_exception(self): diff --git a/manila/tests/test_utils.py b/manila/tests/test_utils.py index c8bc59146c..039057b500 100644 --- a/manila/tests/test_utils.py +++ b/manila/tests/test_utils.py @@ -223,7 +223,8 @@ class FakeSSHClient(object): pass def connect(self, ip, port=22, username=None, password=None, - key_filename=None, look_for_keys=None, timeout=10): + key_filename=None, look_for_keys=None, timeout=10, + banner_timeout=10): pass def get_transport(self): @@ -282,7 +283,7 @@ class SSHPoolTestCase(test.TestCase): fake_ssh_client.connect.assert_called_once_with( "127.0.0.1", port=22, username="test", password="test", key_filename=None, look_for_keys=False, - timeout=10) + timeout=10, banner_timeout=10) def test_create_ssh_with_key(self): path_to_private_key = "/fakepath/to/privatekey" @@ -295,7 +296,7 @@ class SSHPoolTestCase(test.TestCase): fake_ssh_client.connect.assert_called_once_with( "127.0.0.1", port=22, username="test", password=None, key_filename=path_to_private_key, look_for_keys=False, - timeout=10) + timeout=10, banner_timeout=10) def test_create_ssh_with_nothing(self): fake_ssh_client = mock.Mock() @@ -306,7 +307,7 @@ class SSHPoolTestCase(test.TestCase): fake_ssh_client.connect.assert_called_once_with( "127.0.0.1", port=22, username="test", password=None, key_filename=None, look_for_keys=True, - timeout=10) + timeout=10, banner_timeout=10) def test_create_ssh_error_connecting(self): attrs = {'connect.side_effect': paramiko.SSHException, } @@ -318,7 +319,7 @@ class SSHPoolTestCase(test.TestCase): fake_ssh_client.connect.assert_called_once_with( "127.0.0.1", port=22, username="test", password=None, key_filename=None, look_for_keys=True, - timeout=10) + timeout=10, banner_timeout=10) def test_closed_reopend_ssh_connections(self): with mock.patch.object(paramiko, "SSHClient", diff --git a/manila/utils.py b/manila/utils.py index 5cc2282825..13218b8658 100644 --- a/manila/utils.py +++ b/manila/utils.py @@ -103,13 +103,25 @@ class SSHPool(pools.Pool): elif self.password: look_for_keys = False try: + LOG.debug("ssh.connect: ip: %s, port: %s, username: %s, " + "password: %s, key_filename: %s, look_for_keys: %s, " + "timeout: %s, banner_timeout: %s", + self.ip, + self.port, + self.login, + self.password, + self.path_to_private_key, + look_for_keys, + self.conn_timeout, + self.conn_timeout) ssh.connect(self.ip, port=self.port, username=self.login, password=self.password, key_filename=self.path_to_private_key, look_for_keys=look_for_keys, - timeout=self.conn_timeout) + timeout=self.conn_timeout, + banner_timeout=self.conn_timeout) if self.conn_timeout: transport = ssh.get_transport() transport.set_keepalive(self.conn_timeout)