Delete zombie amphorae when detected

Zombie amphorae will be deleted by the system to lighten the load on
operators if the amphora record is still in the database.

Story: 2003912
Task: 26800

Change-Id: If133a3d36a9381bcca9f7d00f5c1531885907940

octavia/controller/healthmanager/health_drivers/update_db.py

@@ -176,10 +176,28 @@ class UpdateHealthDb(update_base.HealthUpdateBase):
                 LOG.debug('Received a health heartbeat from amphora {0} with '
                           'IP {1} that should not exist. This amphora may be '
                           'in the process of being deleted, in which case you '
-                          'will only see this message a few times. However if '
-                          'it is repeating this amphora should be manually '
-                          'deleted.'.format(health['id'], srcaddr))
-                return
+                          'will only see this message a few '
+                          'times'.format(health['id'], srcaddr))
+                if not amp:
+                    LOG.warning('The amphora {0} with IP {1} is missing from '
+                                'the DB, so it cannot be automatically '
+                                'deleted (the compute_id is unknown). An '
+                                'operator must manually delete it from the '
+                                'compute service.'.format(health['id'],
+                                                          srcaddr))
+                    return
+                # delete the amp right there
+                try:
+                    compute = stevedore_driver.DriverManager(
+                        namespace='octavia.compute.drivers',
+                        name=CONF.controller_worker.compute_driver,
+                        invoke_on_load=True
+                    ).driver
+                    compute.delete(amp.compute_id)
+                    return
+                except Exception as e:
+                    LOG.info("Error deleting amp {0} with IP {1}".format(
+                        health['id'], srcaddr), e)
             expected_listener_count = 0
 
         listeners = health['listeners']

octavia/tests/unit/controller/healthmanager/health_drivers/test_update_db.py

@@ -1021,6 +1021,17 @@ class TestUpdateHealthDb(base.TestCase):
                         self.session_mock, member_id,
                         operating_status=constants.ONLINE)
 
+    @mock.patch('stevedore.driver.DriverManager.driver')
+    def test_update_health_zombie(self, mock_driver):
+        health = {"id": self.FAKE_UUID_1, "listeners": {}}
+
+        self.amphora_repo.get_lb_for_health_update.return_value = None
+        amp_mock = mock.MagicMock()
+        self.amphora_repo.get.return_value = amp_mock
+        self.hm.update_health(health, '192.0.2.1')
+        mock_driver.delete.assert_called_once_with(
+            amp_mock.compute_id)
+
     def test_update_health_no_status_change(self):
         health = {
             "id": self.FAKE_UUID_1,

releasenotes/notes/zombie_amp-1b435eb66643dab8.yaml

@@ -0,0 +1,12 @@
+---
+fixes:
+  - |
+    This will automatically nova delete zombie amphora when they
+    are detected by Octavia. Zombie amphorae are amphorae which
+    report health messages but appear DELETED in Octavia's
+    database.
+other:
+  - |
+    Processing zombie amphora is already expensive and this adds
+    another step which could increase the load on Octavia Health
+    Manager, especially during Nova API slowness.