Add test_service_with_not_ready_endpoints test
This test case automates the issue described in BZ https://bugzilla.redhat.com/show_bug.cgi?id=1980957. The new test case: - creates a deployment with a failing readiness probe so all the pods are not ready - creates a service for the deployment (where the endpoints are not ready) - it checks Kuryr pods are not restarted Change-Id: Id19f85625901b35900f9441aace8fe8938cb1d68
This commit is contained in:
parent
d9a93bcba9
commit
2ffdd2f475
|
@ -284,13 +284,21 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
|
|||
return False
|
||||
|
||||
@classmethod
|
||||
def get_pods_ready_num(cls, namespace="default",
|
||||
label="", num_pods=1):
|
||||
def check_pods_ready_num(cls, namespace="default",
|
||||
label="", num_pods=1):
|
||||
pods = cls.get_pod_name_list(namespace=namespace,
|
||||
label_selector=label)
|
||||
ready_pods = sum([cls.get_readiness_state(p) for p in pods])
|
||||
return (num_pods == ready_pods)
|
||||
|
||||
@classmethod
|
||||
def check_pods_status_num(cls, namespace="default", label="", num_pods=1,
|
||||
status="Running"):
|
||||
pods = cls.get_pod_name_list(namespace=namespace,
|
||||
label_selector=label)
|
||||
status_pods = sum([cls.get_pod_status(p) == status for p in pods])
|
||||
return (num_pods == status_pods)
|
||||
|
||||
@classmethod
|
||||
def get_pod_readiness(cls, pod_name, namespace="default"):
|
||||
LOG.info("Checking if pod {} is ready".format(pod_name))
|
||||
|
@ -406,7 +414,7 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
|
|||
|
||||
def create_deployment(self, deployment_name=None, api_version="apps/v1",
|
||||
kind="Deployment", namespace="default",
|
||||
labels={"app": "demo"}):
|
||||
labels={"app": "demo"}, failing_probe=False):
|
||||
api_instance = kubernetes.client.AppsV1Api()
|
||||
if not deployment_name:
|
||||
deployment_name = data_utils.rand_name(prefix='kuryr-deployment')
|
||||
|
@ -418,6 +426,12 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
|
|||
{"image": "quay.io/kuryr/demo",
|
||||
"name": 'demo',
|
||||
"ports": [{"containerPort": 8080}]}]}}
|
||||
if failing_probe:
|
||||
for container in template["spec"]["containers"]:
|
||||
container["readinessProbe"] = {"httpGet": {"path": "/healthz",
|
||||
"port": 8089},
|
||||
"initialDelaySeconds": 2,
|
||||
"timeoutSeconds": 1}
|
||||
spec = k8s_client.V1DeploymentSpec(
|
||||
replicas=3,
|
||||
selector={"matchLabels": {"app": "demo"}}, template=template)
|
||||
|
@ -440,7 +454,7 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
|
|||
# NOTE(juriarte): Wait timeout increased from 180 to 300 in order to
|
||||
# give the pods time to transition to ready status in the gates (and
|
||||
# slow environments).
|
||||
self.wait_for_status(300, 15, self.get_pods_ready_num,
|
||||
self.wait_for_status(300, 15, self.check_pods_ready_num,
|
||||
namespace=namespace, label=label,
|
||||
num_pods=replicas)
|
||||
|
||||
|
@ -1622,3 +1636,14 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
|
|||
raise lib_exc.TimeoutException("Expected num of members is %s but"
|
||||
" actual is %s" % (expected_members,
|
||||
num_members))
|
||||
|
||||
@classmethod
|
||||
def get_pod_containers_restarts(cls, pod_names, namespace='default'):
|
||||
containers = {}
|
||||
for pod_name in pod_names:
|
||||
containers[pod_name] = {}
|
||||
pod = cls.k8s_client.CoreV1Api().read_namespaced_pod(pod_name,
|
||||
namespace)
|
||||
for container in pod.status.container_statuses:
|
||||
containers[pod_name][container.name] = container.restart_count
|
||||
return containers
|
||||
|
|
|
@ -19,5 +19,7 @@ POD_AFFINITY = {'requiredDuringSchedulingIgnoredDuringExecution': [
|
|||
'topologyKey': 'kubernetes.io/hostname'}]}
|
||||
TIME_TO_APPLY_SGS = 30
|
||||
POD_STATUS_RETRIES = 240
|
||||
POD_CHECK_TIMEOUT = 240
|
||||
POD_CHECK_SLEEP_TIME = 5
|
||||
NS_TIMEOUT = 600
|
||||
REPETITIONS_PER_BACKEND = 10
|
||||
|
|
|
@ -18,10 +18,12 @@ import kubernetes
|
|||
from oslo_log import log as logging
|
||||
from tempest import config
|
||||
from tempest.lib.common.utils import data_utils
|
||||
from tempest.lib.common.utils import test_utils
|
||||
from tempest.lib import decorators
|
||||
from tempest.lib import exceptions as lib_exc
|
||||
|
||||
from kuryr_tempest_plugin.tests.scenario import base
|
||||
from kuryr_tempest_plugin.tests.scenario import consts
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
CONF = config.CONF
|
||||
|
@ -340,3 +342,59 @@ class TestLoadBalancerReconciliationScenario(base.BaseKuryrScenarioTest):
|
|||
# if there is a connectivity now, that means the LoadBalancer
|
||||
# is reconciled
|
||||
self.check_service_internal_connectivity(service_name=service_name)
|
||||
|
||||
|
||||
class TestServiceWithNotReadyEndpoints(base.BaseKuryrScenarioTest):
|
||||
|
||||
@classmethod
|
||||
def skip_checks(cls):
|
||||
super(TestServiceWithNotReadyEndpoints, cls).skip_checks()
|
||||
if not CONF.kuryr_kubernetes.service_tests_enabled:
|
||||
raise cls.skipException("Service tests are not enabled")
|
||||
if not CONF.kuryr_kubernetes.containerized:
|
||||
raise cls.skipException("Only runs on containerized setups")
|
||||
|
||||
@decorators.idempotent_id('bddf5441-1244-450d-a125-b5fdcfa1a7b0')
|
||||
def test_service_with_not_ready_endpoints(self):
|
||||
# Create a deployment with a failing probe
|
||||
deployment_name, _ = self.create_deployment(failing_probe=True)
|
||||
|
||||
# Wait until the deployment's pods are running
|
||||
res = test_utils.call_until_true(
|
||||
self.check_pods_status_num, consts.POD_CHECK_TIMEOUT*3,
|
||||
consts.POD_CHECK_SLEEP_TIME, namespace='default', label='app=demo',
|
||||
num_pods=3)
|
||||
self.assertTrue(res, 'Timed out waiting for pods to be running')
|
||||
|
||||
# Wait until the pods are not ready
|
||||
self.wait_for_status(
|
||||
consts.POD_CHECK_TIMEOUT*3, consts.POD_CHECK_SLEEP_TIME,
|
||||
self.check_pods_ready_num, namespace='default', label='app=demo',
|
||||
num_pods=0)
|
||||
|
||||
# Get current Kuryr pods restart count (for a later comparison)
|
||||
controller_pods = self.get_controller_pod_names()
|
||||
container_restarts_before = self.get_pod_containers_restarts(
|
||||
pod_names=controller_pods,
|
||||
namespace=CONF.kuryr_kubernetes.kube_system_namespace)
|
||||
|
||||
# Create a service
|
||||
service_name, _ = self.create_service(pod_label={"app": "demo"},
|
||||
spec_type='ClusterIP')
|
||||
self.addCleanup(self.delete_service, service_name)
|
||||
|
||||
# Check Kuryr pods are not restarted
|
||||
self.check_controller_pod_status_for_time_period(
|
||||
retry_attempts=10,
|
||||
time_between_attempts=3)
|
||||
|
||||
# Get current Kuryr pods restart count
|
||||
container_restarts_after = self.get_pod_containers_restarts(
|
||||
pod_names=controller_pods,
|
||||
namespace=CONF.kuryr_kubernetes.kube_system_namespace)
|
||||
|
||||
# Compare Kuryr pods restart count with previously stored data
|
||||
self.assertEqual(container_restarts_before, container_restarts_after,
|
||||
"Kuryr controller pod(s) were restarted during the "
|
||||
"service creation, expected: %s, obtained: %s" %
|
||||
(container_restarts_before, container_restarts_after))
|
||||
|
|
Loading…
Reference in New Issue