From 33594b870297aa88f15d73c221b6aa04709afe6d Mon Sep 17 00:00:00 2001
From: Luis Tomas Bolivar <ltomasbo@redhat.com>
Date: Thu, 14 Feb 2019 10:57:21 +0100
Subject: [PATCH] Ensure kuryr-controller recover from lb in ERROR status

This patch ensures kuryr controller can recover from the situation
when the created lb goes into ERROR status (instead of ACTIVE). Now,
when the kuryr-controller finds the created lb, it checks its
provisioning_status and if in ERROR status it will delete it and
ensure a ResourceNotReady exception is triggered so that a new
creation action is triggered after the deletion of the lb in ERROR
status.

Closes-Bug: 1815880
Change-Id: I3f5de710a5ff37dee05f5f8826cb37c343141a08
---
 .../controller/drivers/lbaasv2.py             |  6 ++++
 .../unit/controller/drivers/test_lbaasv2.py   | 28 ++++++++++++++++++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/kuryr_kubernetes/controller/drivers/lbaasv2.py b/kuryr_kubernetes/controller/drivers/lbaasv2.py
index 629502aee..9692fcfdc 100644
--- a/kuryr_kubernetes/controller/drivers/lbaasv2.py
+++ b/kuryr_kubernetes/controller/drivers/lbaasv2.py
@@ -74,6 +74,8 @@ class LBaaSv2Driver(base.LBaaSDriver):
         if not response:
             # NOTE(ivc): load balancer was present before 'create', but got
             # deleted externally between 'create' and 'find'
+            # NOTE(ltomasbo): or it is in ERROR status, so we deleted and
+            # trigger the retry
             raise k_exc.ResourceNotReady(request)
 
         return response
@@ -521,6 +523,10 @@ class LBaaSv2Driver(base.LBaaSDriver):
             loadbalancer.id = response['loadbalancers'][0]['id']
             loadbalancer.port_id = self._get_vip_port(loadbalancer).get("id")
             loadbalancer.provider = response['loadbalancers'][0]['provider']
+            if (response['loadbalancers'][0]['provisioning_status'] ==
+                    'ERROR'):
+                self.release_loadbalancer(loadbalancer)
+                return None
         except (KeyError, IndexError):
             return None
 
diff --git a/kuryr_kubernetes/tests/unit/controller/drivers/test_lbaasv2.py b/kuryr_kubernetes/tests/unit/controller/drivers/test_lbaasv2.py
index a6ff73947..4545022c0 100644
--- a/kuryr_kubernetes/tests/unit/controller/drivers/test_lbaasv2.py
+++ b/kuryr_kubernetes/tests/unit/controller/drivers/test_lbaasv2.py
@@ -343,7 +343,8 @@ class TestLBaaSv2Driver(test_base.TestCase):
             provider='haproxy', security_groups=[])
         loadbalancer_id = '00EE9E11-91C2-41CF-8FD4-7970579E5C4C'
         resp = {'loadbalancers': [{'id': loadbalancer_id,
-                                   'provider': 'haproxy'}]}
+                                   'provider': 'haproxy',
+                                   'provisioning_status': 'ACTIVE'}]}
         lbaas.list_loadbalancers.return_value = resp
         m_driver._get_vip_port.return_value = {'id': mock.sentinel.port_id}
 
@@ -357,6 +358,7 @@ class TestLBaaSv2Driver(test_base.TestCase):
             self.assertEqual(getattr(loadbalancer, attr),
                              getattr(ret, attr))
         self.assertEqual(loadbalancer_id, ret.id)
+        m_driver.release_loadbalancer.assert_not_called()
 
     def test_find_loadbalancer_not_found(self):
         lbaas = self.useFixture(k_fix.MockLBaaSClient()).client
@@ -375,6 +377,30 @@ class TestLBaaSv2Driver(test_base.TestCase):
             vip_address=str(loadbalancer.ip),
             vip_subnet_id=loadbalancer.subnet_id)
         self.assertIsNone(ret)
+        m_driver.release_loadbalancer.assert_not_called()
+
+    def test_find_loadbalancer_error(self):
+        lbaas = self.useFixture(k_fix.MockLBaaSClient()).client
+        cls = d_lbaasv2.LBaaSv2Driver
+        m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver)
+        loadbalancer = obj_lbaas.LBaaSLoadBalancer(
+            name='TEST_NAME', project_id='TEST_PROJECT', ip='1.2.3.4',
+            subnet_id='D3FA400A-F543-4B91-9CD3-047AF0CE42D1')
+        loadbalancer_id = '00EE9E11-91C2-41CF-8FD4-7970579E5C4C'
+        resp = {'loadbalancers': [{'id': loadbalancer_id,
+                                   'provider': 'haproxy',
+                                   'provisioning_status': 'ERROR'}]}
+        lbaas.list_loadbalancers.return_value = resp
+        m_driver._get_vip_port.return_value = {'id': mock.sentinel.port_id}
+
+        ret = cls._find_loadbalancer(m_driver, loadbalancer)
+        lbaas.list_loadbalancers.assert_called_once_with(
+            name=loadbalancer.name,
+            project_id=loadbalancer.project_id,
+            vip_address=str(loadbalancer.ip),
+            vip_subnet_id=loadbalancer.subnet_id)
+        self.assertIsNone(ret)
+        m_driver.release_loadbalancer.assert_called_once()
 
     def test_create_listener(self):
         lbaas = self.useFixture(k_fix.MockLBaaSClient()).client