Services: Rollback openstack resources in case of annotation failure

Upon K8S service creation the LBaaS handler creates all LB resources
at neutron (LB,Listener,Pool,etc) and store them at K8S resource
 using annotation.
When K8S service is deleted, the LBaaS handler retrieves LB
resources details from annotation and release them at neutron.

This patch handles the case in which K8S service resource was deleted
before LBaaS handler stored openstack resource details.

Closes-Bug: 1748890

Change-Id: Iea806d32c99cd3cf51a832b576ff4054fc522bd3
This commit is contained in:
Yossi Boaron 2018-02-21 23:54:52 +02:00
parent 48baed57d9
commit 2e6c7eaae7
5 changed files with 75 additions and 4 deletions

View File

@ -246,10 +246,19 @@ class LoadBalancerHandler(k8s_base.ResourceEventHandler):
# required to deal with such situations (e.g. cleanup, or skip
# failing items, or validate configuration) to prevent annotation
# being out of sync with the actual Neutron state.
self._set_lbaas_state(endpoints, lbaas_state)
try:
self._set_lbaas_state(endpoints, lbaas_state)
except k_exc.K8sResourceNotFound:
# Note(yboaron) It's impossible to store neutron resources
# in K8S object since object was deleted. In that case
# we should rollback all neutron resources.
LOG.debug("LoadBalancerHandler failed to store Openstack "
"resources in K8S object (not found)")
self.on_deleted(endpoints, lbaas_state)
def on_deleted(self, endpoints):
lbaas_state = self._get_lbaas_state(endpoints)
def on_deleted(self, endpoints, lbaas_state=None):
if lbaas_state is None:
lbaas_state = self._get_lbaas_state(endpoints)
if not lbaas_state:
return
# NOTE(ivc): deleting pool deletes its members

View File

@ -28,6 +28,12 @@ class ResourceNotReady(Exception):
% resource)
class K8sResourceNotFound(K8sClientException):
def __init__(self, resource):
super(K8sResourceNotFound, self).__init__("Resource not "
"found: %r" % resource)
class CNIError(Exception):
pass

View File

@ -145,7 +145,11 @@ class K8sClient(object):
"content: %(content)s, text: %(text)s"
% {'headers': response.headers,
'content': response.content, 'text': response.text})
raise exc.K8sClientException(response.text)
if response.status_code == requests.codes.not_found:
raise exc.K8sResourceNotFound(response.text)
else:
raise exc.K8sClientException(response.text)
def watch(self, path):
params = {'watch': 'true'}

View File

@ -423,6 +423,31 @@ class TestLoadBalancerHandler(test_base.TestCase):
m_handler._set_lbaas_state.assert_called_once_with(
endpoints, lbaas_state)
def test_on_present_rollback(self):
lbaas_spec = mock.sentinel.lbaas_spec
lbaas_state = mock.sentinel.lbaas_state
endpoints = mock.sentinel.endpoints
m_handler = mock.Mock(spec=h_lbaas.LoadBalancerHandler)
m_handler._get_lbaas_spec.return_value = lbaas_spec
m_handler._should_ignore.return_value = False
m_handler._get_lbaas_state.return_value = lbaas_state
m_handler._sync_lbaas_members.return_value = True
m_handler._set_lbaas_state.side_effect = (
k_exc.K8sResourceNotFound('ep'))
h_lbaas.LoadBalancerHandler.on_present(m_handler, endpoints)
m_handler._get_lbaas_spec.assert_called_once_with(endpoints)
m_handler._should_ignore.assert_called_once_with(endpoints, lbaas_spec)
m_handler._get_lbaas_state.assert_called_once_with(endpoints)
m_handler._sync_lbaas_members.assert_called_once_with(
endpoints, lbaas_state, lbaas_spec)
m_handler._set_lbaas_state.assert_called_once_with(
endpoints, lbaas_state)
m_handler.on_deleted.assert_called_once_with(
endpoints, lbaas_state)
@mock.patch('kuryr_kubernetes.objects.lbaas'
'.LBaaSServiceSpec')
def test_on_deleted(self, m_svc_spec_ctor):

View File

@ -280,6 +280,33 @@ class TestK8sClient(test_base.TestCase):
headers=mock.ANY,
cert=(None, None), verify=False)
@mock.patch('itertools.count')
@mock.patch('requests.patch')
def test_annotate_resource_not_found(self, m_patch, m_count):
m_count.return_value = list(range(1, 5))
path = '/test'
annotations = {'a1': 'v1', 'a2': 'v2'}
resource_version = "123"
annotate_obj = {'metadata': {
'annotations': annotations,
'resourceVersion': resource_version}}
annotate_data = jsonutils.dumps(annotate_obj, sort_keys=True)
m_resp_not_found = mock.MagicMock()
m_resp_not_found.ok = False
m_resp_not_found.status_code = requests.codes.not_found
m_patch.return_value = m_resp_not_found
self.assertRaises(exc.K8sResourceNotFound,
self.client.annotate,
path,
annotations,
resource_version=resource_version)
m_patch.assert_called_once_with(self.base_url + path,
data=annotate_data,
headers=mock.ANY,
cert=(None, None), verify=False)
@mock.patch('requests.get')
def test_watch(self, m_get):
path = '/test'