openstack
/
kuryr-kubernetes
Code Issues Proposed changes

Adds CRDs readiness checks to Kuryr-Controller 

This patch adds health checks to Kuryr-Controller to
ensure that the CRDs that it depends on are defined
in the cluster.

Partially implements: blueprint k8s-network-policies
Change-Id: Ie3b6b6900d7a67402bddcca89b18dea35cc86992
This commit is contained in:
Maysa Macedo 2018-11-13 13:19:59 +03:00
parent c76497725f
commit 72b67223d8
3 changed files with 139 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
kuryr_kubernetes/constants.py
View File

@ -17,6 +17,8 @@ K8S_API_BASE = '/api/v1'
K8S_API_NAMESPACES = K8S_API_BASE + '/namespaces'
K8S_API_CRD = '/apis/openstack.org/v1'
K8S_API_CRD_NAMESPACES = K8S_API_CRD + '/namespaces'
K8S_API_CRD_KURYRNETS = K8S_API_CRD + '/kuryrnets'
K8S_API_CRD_KURYRNETPOLICIES = K8S_API_CRD + '/kuryrnetpolicies'
K8S_API_POLICIES = '/apis/networking.k8s.io/v1/networkpolicies'
K8S_API_NPWG_CRD = '/apis/k8s.cni.cncf.io/v1'

19
kuryr_kubernetes/controller/managers/health.py
View File

@ -23,6 +23,7 @@ from kuryr.lib._i18n import _
from kuryr.lib import config as kuryr_config
from kuryr.lib import utils
from kuryr_kubernetes import clients
from kuryr_kubernetes import constants as k_const
from kuryr_kubernetes import exceptions as exc
from kuryr_kubernetes.handlers import health as h_health
@ -57,6 +58,16 @@ class HealthServer(object):
'/alive', methods=['GET'], view_func=self.liveness_status)
self.headers = {'Connection': 'close'}
def _has_kuryr_crd(self, crd_url):
k8s = clients.get_kubernetes_client()
try:
k8s.get(crd_url, json=False, headers={'Connection': 'close'})
except exc.K8sClientException:
LOG.exception("Kubernetes Client Exception fetching"
" CRD. %s" % exc.K8sClientException)
return False
return True
def readiness_status(self):
data = 'ok'
@ -85,6 +96,14 @@ class HealthServer(object):
LOG.exception(error_message)
return error_message, httplib.INTERNAL_SERVER_ERROR, self.headers
crds = [k_const.K8S_API_CRD_KURYRNETS,
k_const.K8S_API_CRD_KURYRNETPOLICIES]
for crd in crds:
if not self._has_kuryr_crd(crd):
error_msg = "Error when processing '%s' CRD request." % crd
LOG.error(error_msg)
return error_msg, httplib.INTERNAL_SERVER_ERROR, self.headers
LOG.info('Kuryr Controller readiness verified.')
return data, httplib.OK, self.headers

126
kuryr_kubernetes/tests/unit/controller/managers/test_health.py
View File

@ -12,9 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from kuryr_kubernetes import constants as k_const
from kuryr_kubernetes.controller.managers import health
from kuryr_kubernetes import exceptions as k_exc
from kuryr_kubernetes.handlers import health as h_health
from kuryr_kubernetes.tests import base
from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix
import mock
from oslo_config import cfg as oslo_cfg
@ -32,6 +35,8 @@ class TestHealthServer(base.TestCase):
self.srv.application.testing = True
self.test_client = self.srv.application.test_client()
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'_has_kuryr_crd')
@mock.patch('os.path.exists')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_neutron_connection')
@ -39,20 +44,25 @@ class TestHealthServer(base.TestCase):
'verify_keystone_connection')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_k8s_connection')
def test_read(self, m_verify_k8s_conn, m_verify_keystone_conn,
m_verify_neutron_conn, m_exist):
def test_readiness(self, m_verify_k8s_conn, m_verify_keystone_conn,
m_verify_neutron_conn, m_exist,
m_has_kuryr_crd):
m_has_kuryr_crd.side_effect = [True, True]
m_verify_k8s_conn.return_value = True, 200
m_exist.return_value = True
resp = self.test_client.get('/ready')
m_verify_k8s_conn.assert_called_once()
m_verify_keystone_conn.assert_called_once()
m_verify_neutron_conn.assert_called_once_with()
self.assertEqual(m_has_kuryr_crd.call_count, 2)
self.assertEqual(200, resp.status_code)
self.assertEqual('ok', resp.data.decode())
@mock.patch('os.path.exists')
def test_read_not_found(self, m_exist):
def test_readiness_not_found(self, m_exist):
m_exist.return_value = False
oslo_cfg.CONF.set_override('vif_pool_driver', 'neutron',
group='kubernetes')
@ -62,7 +72,7 @@ class TestHealthServer(base.TestCase):
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_k8s_connection')
@mock.patch('os.path.exists')
def test_read_k8s_error(self, m_exist, m_verify_k8s_conn):
def test_readiness_k8s_error(self, m_exist, m_verify_k8s_conn):
m_exist.return_value = True
m_verify_k8s_conn.return_value = False
resp = self.test_client.get('/ready')
@ -75,8 +85,8 @@ class TestHealthServer(base.TestCase):
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_k8s_connection')
@mock.patch('os.path.exists')
def test_read_unauthorized(self, m_exist, m_verify_k8s_conn,
m_verify_keystone_conn):
def test_readiness_unauthorized(self, m_exist, m_verify_k8s_conn,
m_verify_keystone_conn):
m_exist.return_value = True
m_verify_k8s_conn.return_value = True, 200
m_verify_keystone_conn.side_effect = Exception
@ -92,8 +102,9 @@ class TestHealthServer(base.TestCase):
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_k8s_connection')
@mock.patch('os.path.exists')
def test_read_neutron_error(self, m_exist, m_verify_k8s_conn,
m_verify_keystone_conn, m_verify_neutron_conn):
def test_readiness_neutron_error(self, m_exist, m_verify_k8s_conn,
m_verify_keystone_conn,
m_verify_neutron_conn):
m_exist.return_value = True
m_verify_k8s_conn.return_value = True, 200
m_verify_neutron_conn.side_effect = Exception
@ -102,6 +113,105 @@ class TestHealthServer(base.TestCase):
m_verify_neutron_conn.assert_called_once()
self.assertEqual(500, resp.status_code)
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'_has_kuryr_crd')
@mock.patch('os.path.exists')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_neutron_connection')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_keystone_connection')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_k8s_connection')
def test_readiness_kuryrnet_crd_error(self, m_verify_k8s_conn,
m_verify_keystone_conn,
m_verify_neutron_conn, m_exist,
m_has_kuryr_crd):
kuryrnets_url = k_const.K8S_API_CRD_KURYRNETS
m_has_kuryr_crd.side_effect = [False]
resp = self.test_client.get('/ready')
m_has_kuryr_crd.assert_called_with(kuryrnets_url)
self.assertEqual(m_has_kuryr_crd.call_count, 1)
self.assertEqual(500, resp.status_code)
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'_has_kuryr_crd')
@mock.patch('os.path.exists')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_neutron_connection')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_keystone_connection')
@mock.patch('kuryr_kubernetes.controller.managers.health.HealthServer.'
'verify_k8s_connection')
def test_readiness_kuryrnetpolicy_crd_error(self, m_verify_k8s_conn,
m_verify_keystone_conn,
m_verify_neutron_conn, m_exist,
m_has_kuryr_crd):
kuryrnetpolicies_url = k_const.K8S_API_CRD_KURYRNETPOLICIES
m_has_kuryr_crd.side_effect = [True, False]
resp = self.test_client.get('/ready')
self.assertEqual(m_has_kuryr_crd.call_count, 2)
m_has_kuryr_crd.assert_called_with(kuryrnetpolicies_url)
self.assertEqual(500, resp.status_code)
def test__has_kuryrnet_crd(self):
kuryrnet_crd = {
"apiVersion": "openstack.org/v1",
"items": [
],
"kind": "KuryrNetList",
"metadata": {
"continue": "",
"resourceVersion": "33018",
"selfLink": "/apis/openstack.org/v1/kuryrnets"
}
}
kubernetes = self.useFixture(k_fix.MockK8sClient()).client
kubernetes.get.return_value = kuryrnet_crd
kuryrnets_url = k_const.K8S_API_CRD_KURYRNETS
resp = self.srv._has_kuryr_crd(kuryrnets_url)
self.assertEqual(resp, True)
def test__has_kuryrnetpolicy_crd(self):
kuryrnetpolicies_crd = {
"apiVersion": "openstack.org/v1",
"items": [
],
"kind": "KuryrNetPolicyList",
"metadata": {
"continue": "",
"resourceVersion": "34186",
"selfLink": "/apis/openstack.org/v1/kuryrnetpolicies"
}
}
kubernetes = self.useFixture(k_fix.MockK8sClient()).client
kubernetes.get.return_value = kuryrnetpolicies_crd
kuryrnetpolicies_url = k_const.K8S_API_CRD_KURYRNETPOLICIES
resp = self.srv._has_kuryr_crd(kuryrnetpolicies_url)
self.assertEqual(resp, True)
def test__has_kuryr_crd_error(self):
crds = [k_const.K8S_API_CRD_KURYRNETS,
k_const.K8S_API_CRD_KURYRNETPOLICIES]
for crd_url in crds:
kubernetes = self.useFixture(k_fix.MockK8sClient()).client
kubernetes.get.side_effect = k_exc.K8sClientException
resp = self.srv._has_kuryr_crd(crd_url)
self.assertEqual(resp, False)
kubernetes.get.assert_called_once()
@mock.patch.object(_TestHandler, 'is_healthy')
def test_liveness(self, m_status):
m_status.return_value = True