From 963cb3f5b4bafada773daf5fd48bbf13ce6a3e45 Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jim@acmegating.com>
Date: Tue, 5 Mar 2024 16:05:18 -0800
Subject: [PATCH] gce: clear machine type cache on bad data

We have observed GCE returning bad machine type data which we
then cache.  If that happens, clear the cache to avoid getting
stuck with the bad data.

Change-Id: I32fac2a92d4f9d400fe2db41fffd8d189d097542
---
 nodepool/driver/gce/adapter.py         | 14 +++++++----
 nodepool/tests/unit/test_driver_gce.py | 32 ++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/nodepool/driver/gce/adapter.py b/nodepool/driver/gce/adapter.py
index c16c5c304..52084a733 100644
--- a/nodepool/driver/gce/adapter.py
+++ b/nodepool/driver/gce/adapter.py
@@ -290,10 +290,16 @@ class GceAdapter(statemachine.Adapter):
 
     def _getQuotaForMachineType(self, machine_type):
         mtype = self._getMachineType(machine_type)
-        return QuotaInformation(
-            cores=mtype['guestCpus'],
-            instances=1,
-            ram=mtype['memoryMb'])
+        try:
+            qi = QuotaInformation(
+                cores=mtype['guestCpus'],
+                instances=1,
+                ram=mtype['memoryMb'])
+        except Exception:
+            self.log.exception("Machine type has bad format: %s", mtype)
+            self._getMachineType.cache_clear()
+            raise
+        return qi
 
     def _getInstance(self, hostname):
         for instance in self._listInstances():
diff --git a/nodepool/tests/unit/test_driver_gce.py b/nodepool/tests/unit/test_driver_gce.py
index bce78b8ce..0ed3094ea 100644
--- a/nodepool/tests/unit/test_driver_gce.py
+++ b/nodepool/tests/unit/test_driver_gce.py
@@ -21,6 +21,7 @@ from functools import wraps
 
 import yaml
 import fixtures
+import testtools
 
 import googleapiclient.discovery
 import googleapiclient.errors
@@ -342,3 +343,34 @@ class TestDriverGce(tests.DBTestCase):
 
     def test_gce_machine(self):
         self._test_gce_machine('debian-stretch-f1-micro')
+
+    @_test_with_pool
+    def test_machine_type_error(self, pool):
+        self.startPool(pool)
+        self._wait_for_provider(pool, 'gcloud-provider')
+
+        # Just return an empty dict
+        def fakeMachineTypeFactory():
+            class FakeQuery:
+                def execute(self):
+                    return {}
+
+            class FakeMachineTypes:
+                def get(**kw):
+                    return FakeQuery()
+            return FakeMachineTypes
+
+        adapter = pool.getProviderManager('gcloud-provider').adapter
+
+        # Test the working path
+        qi = adapter._getQuotaForMachineType('f1-micro')
+        self.assertEqual(qi.quota['compute']['cores'], 1)
+
+        # Return some garbage data
+        self.patch(adapter.compute, 'machineTypes', fakeMachineTypeFactory)
+        with testtools.ExpectedException(KeyError):
+            adapter._getQuotaForMachineType('n1-standard-2')
+
+        # Verify we don't get the cached value from earlier
+        with testtools.ExpectedException(KeyError):
+            adapter._getQuotaForMachineType('f1-micro')