diff --git a/nodepool/driver/static/handler.py b/nodepool/driver/static/handler.py index ac204a4be..10ad0eb23 100644 --- a/nodepool/driver/static/handler.py +++ b/nodepool/driver/static/handler.py @@ -44,3 +44,6 @@ class StaticNodeRequestHandler(NodeRequestHandler): def launchesComplete(self): # We don't wait on a launch since we never actually launch. return True + + def checkReusableNode(self, node): + return self.manager.checkNodeLiveness(node) diff --git a/nodepool/driver/static/provider.py b/nodepool/driver/static/provider.py index 64d1cdd5c..a76f2ad57 100644 --- a/nodepool/driver/static/provider.py +++ b/nodepool/driver/static/provider.py @@ -78,6 +78,27 @@ class StaticNodeProvider(Provider): nodes.append(node) return nodes + def checkNodeLiveness(self, node): + static_node = self.poolNodes().get(node.hostname) + if static_node is None: + return False + + try: + nodeutils.nodescan(static_node["name"], + port=static_node["connection-port"], + timeout=static_node["timeout"], + gather_hostkeys=False) + return True + except Exception: + self.log.exception("Failed to connect to node %s:", + static_node["name"]) + try: + self.deregisterNode(count=1, node_name=static_node["name"]) + except Exception: + self.log.exception("Couldn't deregister static node:") + + return False + def getRegisteredNodeHostnames(self): ''' Get hostnames for all registered static nodes. diff --git a/nodepool/tests/test_driver_static.py b/nodepool/tests/test_driver_static.py index 0e863ac4e..c36c795d6 100644 --- a/nodepool/tests/test_driver_static.py +++ b/nodepool/tests/test_driver_static.py @@ -14,6 +14,7 @@ # limitations under the License. import logging +import mock import os from nodepool import config as nodepool_config @@ -278,6 +279,32 @@ class TestDriverStatic(tests.DBTestCase): self.assertEqual(len(new_nodes), 1) self.assertEqual(nodes[0].hostname, new_nodes[0].hostname) + def test_liveness_check(self): + ''' + Test liveness check during request handling. + ''' + configfile = self.setup_config('static-basic.yaml') + pool = self.useNodepool(configfile, watermark_sleep=1) + pool.start() + nodes = self.waitForNodes('fake-label') + self.assertEqual(len(nodes), 1) + + req = zk.NodeRequest() + req.state = zk.REQUESTED + req.node_types.append('fake-label') + + with mock.patch("nodepool.nodeutils.nodescan") as nodescan_mock: + nodescan_mock.side_effect = OSError + self.zk.storeNodeRequest(req) + self.waitForNodeDeletion(nodes[0]) + + self.log.debug("Waiting for request %s", req.id) + req = self.waitForNodeRequest(req) + + self.assertEqual(req.state, zk.FULFILLED) + self.assertEqual(len(req.nodes), 1) + self.assertNotEqual(req.nodes[0], nodes[0].id) + def test_missing_static_node(self): """Test that a missing static node is added""" configfile = self.setup_config('static-2-nodes.yaml')