Ensure controller healthchecks passes without CRDs

This patch ensures the controller healthchecks do not set the
controller as not Ready due to missing CRDs when deploying without
namespace and/or policy handlers. In that case the CRDs are not needed

Closes-Bug: 1808966
Change-Id: I685f9a47605da86504619983848b8ef73d71b332
This commit is contained in:
Luis Tomas Bolivar 2018-12-21 12:02:54 +01:00
parent 064b734679
commit 05cdb9cd4f
6 changed files with 84 additions and 129 deletions

View File

@ -111,8 +111,13 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
self._del_kuryrnet_crd(net_crd_id)
@MEMOIZE
def is_ready(self, quota):
if not utils.has_kuryr_crd(constants.K8S_API_CRD_KURYRNETS):
return False
return self._check_quota(quota)
@MEMOIZE
def _check_quota(self, quota):
neutron = clients.get_neutron_client()
resources = {'subnet': neutron.list_subnets,
'network': neutron.list_networks,

View File

@ -98,8 +98,13 @@ class NetworkPolicyHandler(k8s_base.ResourceEventHandler):
self._drv_policy.release_network_policy(netpolicy_crd)
@MEMOIZE
def is_ready(self, quota):
if not utils.has_kuryr_crd(k_const.K8S_API_CRD_KURYRNETPOLICIES):
return False
return self._check_quota(quota)
@MEMOIZE
def _check_quota(self, quota):
neutron = clients.get_neutron_client()
sg_quota = quota['security_group']
sg_func = neutron.list_security_groups

View File

@ -24,7 +24,6 @@ from kuryr.lib import config as kuryr_config
from kuryr.lib import utils
from kuryr_kubernetes import clients
from kuryr_kubernetes import config
from kuryr_kubernetes import constants as k_const
from kuryr_kubernetes import exceptions as exc
from kuryr_kubernetes.handlers import health as h_health
@ -70,16 +69,6 @@ class HealthServer(object):
return False
return True
def _has_kuryr_crd(self, crd_url):
k8s = clients.get_kubernetes_client()
try:
k8s.get(crd_url, json=False, headers={'Connection': 'close'})
except exc.K8sClientException:
LOG.exception("Kubernetes Client Exception fetching"
" CRD. %s" % exc.K8sClientException)
return False
return True
def readiness_status(self):
data = 'ok'
@ -102,14 +91,6 @@ class HealthServer(object):
LOG.exception(error_message)
return error_message, httplib.INTERNAL_SERVER_ERROR, self.headers
crds = [k_const.K8S_API_CRD_KURYRNETS,
k_const.K8S_API_CRD_KURYRNETPOLICIES]
for crd in crds:
if not self._has_kuryr_crd(crd):
error_msg = "Error when processing '%s' CRD request." % crd
LOG.error(error_msg)
return error_msg, httplib.INTERNAL_SERVER_ERROR, self.headers
try:
if not self._components_ready():
return '', httplib.INTERNAL_SERVER_ERROR, self.headers

View File

@ -12,9 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from kuryr_kubernetes import constants as k_const
from kuryr_kubernetes.controller.managers import health
from kuryr_kubernetes import exceptions as k_exc
from kuryr_kubernetes.handlers import health as h_health
from kuryr_kubernetes.tests import base
from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix
@ -56,16 +54,13 @@ class TestHealthServer(base.TestCase):
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'_components_ready')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'_has_kuryr_crd')
@mock.patch('os.path.exists')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_keystone_connection')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_k8s_connection')
def test_readiness(self, m_verify_k8s_conn, m_verify_keystone_conn,
m_exist, m_has_kuryr_crd, m_components_ready):
m_has_kuryr_crd.side_effect = [True, True]
m_exist, m_components_ready):
m_verify_k8s_conn.return_value = True, 200
m_exist.return_value = True
m_components_ready.return_value = True
@ -74,7 +69,6 @@ class TestHealthServer(base.TestCase):
m_verify_k8s_conn.assert_called_once()
m_verify_keystone_conn.assert_called_once()
self.assertEqual(m_has_kuryr_crd.call_count, 2)
m_components_ready.assert_called_once()
self.assertEqual(200, resp.status_code)
@ -114,48 +108,8 @@ class TestHealthServer(base.TestCase):
m_verify_keystone_conn.assert_called_once()
self.assertEqual(500, resp.status_code)
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'_has_kuryr_crd')
@mock.patch('os.path.exists')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_keystone_connection')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_k8s_connection')
def test_readiness_kuryrnet_crd_error(self, m_verify_k8s_conn,
m_verify_keystone_conn,
m_exist, m_has_kuryr_crd):
kuryrnets_url = k_const.K8S_API_CRD_KURYRNETS
m_has_kuryr_crd.side_effect = [False]
resp = self.test_client.get('/ready')
m_has_kuryr_crd.assert_called_with(kuryrnets_url)
self.assertEqual(m_has_kuryr_crd.call_count, 1)
self.assertEqual(500, resp.status_code)
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'_has_kuryr_crd')
@mock.patch('os.path.exists')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_keystone_connection')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_k8s_connection')
def test_readiness_kuryrnetpolicy_crd_error(self, m_verify_k8s_conn,
m_verify_keystone_conn,
m_exist, m_has_kuryr_crd):
kuryrnetpolicies_url = k_const.K8S_API_CRD_KURYRNETPOLICIES
m_has_kuryr_crd.side_effect = [True, False]
resp = self.test_client.get('/ready')
self.assertEqual(m_has_kuryr_crd.call_count, 2)
m_has_kuryr_crd.assert_called_with(kuryrnetpolicies_url)
self.assertEqual(500, resp.status_code)
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'_components_ready')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'_has_kuryr_crd')
@mock.patch('os.path.exists')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_keystone_connection')
@ -163,8 +117,7 @@ class TestHealthServer(base.TestCase):
'verify_k8s_connection')
def test_readiness_neutron_error(self, m_verify_k8s_conn,
m_verify_keystone_conn,
m_exist, m_has_kuryr_crd,
m_components_ready):
m_exist, m_components_ready):
m_components_ready.side_effect = Exception
resp = self.test_client.get('/ready')
@ -174,8 +127,6 @@ class TestHealthServer(base.TestCase):
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'_components_ready')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'_has_kuryr_crd')
@mock.patch('os.path.exists')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_keystone_connection')
@ -183,8 +134,7 @@ class TestHealthServer(base.TestCase):
'verify_k8s_connection')
def test_readiness_components_ready_error(self, m_verify_k8s_conn,
m_verify_keystone_conn,
m_exist, m_has_kuryr_crd,
m_components_ready):
m_exist, m_components_ready):
m_components_ready.return_value = False
resp = self.test_client.get('/ready')
@ -192,61 +142,6 @@ class TestHealthServer(base.TestCase):
m_components_ready.assert_called_once()
self.assertEqual(500, resp.status_code)
def test__has_kuryrnet_crd(self):
kuryrnet_crd = {
"apiVersion": "openstack.org/v1",
"items": [
],
"kind": "KuryrNetList",
"metadata": {
"continue": "",
"resourceVersion": "33018",
"selfLink": "/apis/openstack.org/v1/kuryrnets"
}
}
kubernetes = self.useFixture(k_fix.MockK8sClient()).client
kubernetes.get.return_value = kuryrnet_crd
kuryrnets_url = k_const.K8S_API_CRD_KURYRNETS
resp = self.srv._has_kuryr_crd(kuryrnets_url)
self.assertEqual(resp, True)
def test__has_kuryrnetpolicy_crd(self):
kuryrnetpolicies_crd = {
"apiVersion": "openstack.org/v1",
"items": [
],
"kind": "KuryrNetPolicyList",
"metadata": {
"continue": "",
"resourceVersion": "34186",
"selfLink": "/apis/openstack.org/v1/kuryrnetpolicies"
}
}
kubernetes = self.useFixture(k_fix.MockK8sClient()).client
kubernetes.get.return_value = kuryrnetpolicies_crd
kuryrnetpolicies_url = k_const.K8S_API_CRD_KURYRNETPOLICIES
resp = self.srv._has_kuryr_crd(kuryrnetpolicies_url)
self.assertEqual(resp, True)
def test__has_kuryr_crd_error(self):
crds = [k_const.K8S_API_CRD_KURYRNETS,
k_const.K8S_API_CRD_KURYRNETPOLICIES]
for crd_url in crds:
kubernetes = self.useFixture(k_fix.MockK8sClient()).client
kubernetes.get.side_effect = k_exc.K8sClientException
resp = self.srv._has_kuryr_crd(crd_url)
self.assertEqual(resp, False)
kubernetes.get.assert_called_once()
@mock.patch.object(_TestHandler, 'is_ready')
def test__components_ready(self, m_status):
neutron = self.useFixture(k_fix.MockNeutronClient()).client

View File

@ -16,6 +16,8 @@ import mock
from os_vif import objects
from oslo_config import cfg
from kuryr_kubernetes import constants as k_const
from kuryr_kubernetes import exceptions as k_exc
from kuryr_kubernetes.objects import vif
from kuryr_kubernetes.tests import base as test_base
from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix
@ -100,3 +102,58 @@ class TestUtils(test_base.TestCase):
result = utils.extract_pod_annotation(d)
self.assertEqual(vif.PodState.obj_name(), result.obj_name())
self.assertEqual(vif_obj, result.default_vif)
def test__has_kuryrnet_crd(self):
kuryrnet_crd = {
"apiVersion": "openstack.org/v1",
"items": [
],
"kind": "KuryrNetList",
"metadata": {
"continue": "",
"resourceVersion": "33018",
"selfLink": "/apis/openstack.org/v1/kuryrnets"
}
}
kubernetes = self.useFixture(k_fix.MockK8sClient()).client
kubernetes.get.return_value = kuryrnet_crd
kuryrnets_url = k_const.K8S_API_CRD_KURYRNETS
resp = utils.has_kuryr_crd(kuryrnets_url)
self.assertEqual(resp, True)
def test__has_kuryrnetpolicy_crd(self):
kuryrnetpolicies_crd = {
"apiVersion": "openstack.org/v1",
"items": [
],
"kind": "KuryrNetPolicyList",
"metadata": {
"continue": "",
"resourceVersion": "34186",
"selfLink": "/apis/openstack.org/v1/kuryrnetpolicies"
}
}
kubernetes = self.useFixture(k_fix.MockK8sClient()).client
kubernetes.get.return_value = kuryrnetpolicies_crd
kuryrnetpolicies_url = k_const.K8S_API_CRD_KURYRNETPOLICIES
resp = utils.has_kuryr_crd(kuryrnetpolicies_url)
self.assertEqual(resp, True)
def test__has_kuryr_crd_error(self):
crds = [k_const.K8S_API_CRD_KURYRNETS,
k_const.K8S_API_CRD_KURYRNETPOLICIES]
for crd_url in crds:
kubernetes = self.useFixture(k_fix.MockK8sClient()).client
kubernetes.get.side_effect = k_exc.K8sClientException
resp = utils.has_kuryr_crd(crd_url)
self.assertEqual(resp, False)
kubernetes.get.assert_called_once()

View File

@ -23,6 +23,7 @@ from oslo_log import log
from oslo_serialization import jsonutils
from kuryr_kubernetes import clients
from kuryr_kubernetes import exceptions
from kuryr_kubernetes.objects import vif
from kuryr_kubernetes import os_vif_util
@ -186,3 +187,14 @@ def is_available(resource, resource_quota, neutron_func):
LOG.error("Quota exceeded for resource: %s", resource)
return False
return True
def has_kuryr_crd(crd_url):
k8s = clients.get_kubernetes_client()
try:
k8s.get(crd_url, json=False, headers={'Connection': 'close'})
except exceptions.K8sClientException:
LOG.exception("Kubernetes Client Exception fetching"
" CRD. %s" % exceptions.K8sClientException)
return False
return True