diff --git a/kuryr_tempest_plugin/tests/scenario/base.py b/kuryr_tempest_plugin/tests/scenario/base.py index 16759d74..9be1ec0b 100644 --- a/kuryr_tempest_plugin/tests/scenario/base.py +++ b/kuryr_tempest_plugin/tests/scenario/base.py @@ -284,13 +284,21 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest): return False @classmethod - def get_pods_ready_num(cls, namespace="default", - label="", num_pods=1): + def check_pods_ready_num(cls, namespace="default", + label="", num_pods=1): pods = cls.get_pod_name_list(namespace=namespace, label_selector=label) ready_pods = sum([cls.get_readiness_state(p) for p in pods]) return (num_pods == ready_pods) + @classmethod + def check_pods_status_num(cls, namespace="default", label="", num_pods=1, + status="Running"): + pods = cls.get_pod_name_list(namespace=namespace, + label_selector=label) + status_pods = sum([cls.get_pod_status(p) == status for p in pods]) + return (num_pods == status_pods) + @classmethod def get_pod_readiness(cls, pod_name, namespace="default"): LOG.info("Checking if pod {} is ready".format(pod_name)) @@ -406,7 +414,7 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest): def create_deployment(self, deployment_name=None, api_version="apps/v1", kind="Deployment", namespace="default", - labels={"app": "demo"}): + labels={"app": "demo"}, failing_probe=False): api_instance = kubernetes.client.AppsV1Api() if not deployment_name: deployment_name = data_utils.rand_name(prefix='kuryr-deployment') @@ -418,6 +426,12 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest): {"image": "quay.io/kuryr/demo", "name": 'demo', "ports": [{"containerPort": 8080}]}]}} + if failing_probe: + for container in template["spec"]["containers"]: + container["readinessProbe"] = {"httpGet": {"path": "/healthz", + "port": 8089}, + "initialDelaySeconds": 2, + "timeoutSeconds": 1} spec = k8s_client.V1DeploymentSpec( replicas=3, selector={"matchLabels": {"app": "demo"}}, template=template) @@ -440,7 +454,7 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest): # NOTE(juriarte): Wait timeout increased from 180 to 300 in order to # give the pods time to transition to ready status in the gates (and # slow environments). - self.wait_for_status(300, 15, self.get_pods_ready_num, + self.wait_for_status(300, 15, self.check_pods_ready_num, namespace=namespace, label=label, num_pods=replicas) @@ -1622,3 +1636,14 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest): raise lib_exc.TimeoutException("Expected num of members is %s but" " actual is %s" % (expected_members, num_members)) + + @classmethod + def get_pod_containers_restarts(cls, pod_names, namespace='default'): + containers = {} + for pod_name in pod_names: + containers[pod_name] = {} + pod = cls.k8s_client.CoreV1Api().read_namespaced_pod(pod_name, + namespace) + for container in pod.status.container_statuses: + containers[pod_name][container.name] = container.restart_count + return containers diff --git a/kuryr_tempest_plugin/tests/scenario/consts.py b/kuryr_tempest_plugin/tests/scenario/consts.py index ffc444a6..9ead5150 100644 --- a/kuryr_tempest_plugin/tests/scenario/consts.py +++ b/kuryr_tempest_plugin/tests/scenario/consts.py @@ -19,5 +19,7 @@ POD_AFFINITY = {'requiredDuringSchedulingIgnoredDuringExecution': [ 'topologyKey': 'kubernetes.io/hostname'}]} TIME_TO_APPLY_SGS = 30 POD_STATUS_RETRIES = 240 +POD_CHECK_TIMEOUT = 240 +POD_CHECK_SLEEP_TIME = 5 NS_TIMEOUT = 600 REPETITIONS_PER_BACKEND = 10 diff --git a/kuryr_tempest_plugin/tests/scenario/test_service.py b/kuryr_tempest_plugin/tests/scenario/test_service.py index fdbaf379..15a3f0ff 100644 --- a/kuryr_tempest_plugin/tests/scenario/test_service.py +++ b/kuryr_tempest_plugin/tests/scenario/test_service.py @@ -18,10 +18,12 @@ import kubernetes from oslo_log import log as logging from tempest import config from tempest.lib.common.utils import data_utils +from tempest.lib.common.utils import test_utils from tempest.lib import decorators from tempest.lib import exceptions as lib_exc from kuryr_tempest_plugin.tests.scenario import base +from kuryr_tempest_plugin.tests.scenario import consts LOG = logging.getLogger(__name__) CONF = config.CONF @@ -340,3 +342,59 @@ class TestLoadBalancerReconciliationScenario(base.BaseKuryrScenarioTest): # if there is a connectivity now, that means the LoadBalancer # is reconciled self.check_service_internal_connectivity(service_name=service_name) + + +class TestServiceWithNotReadyEndpoints(base.BaseKuryrScenarioTest): + + @classmethod + def skip_checks(cls): + super(TestServiceWithNotReadyEndpoints, cls).skip_checks() + if not CONF.kuryr_kubernetes.service_tests_enabled: + raise cls.skipException("Service tests are not enabled") + if not CONF.kuryr_kubernetes.containerized: + raise cls.skipException("Only runs on containerized setups") + + @decorators.idempotent_id('bddf5441-1244-450d-a125-b5fdcfa1a7b0') + def test_service_with_not_ready_endpoints(self): + # Create a deployment with a failing probe + deployment_name, _ = self.create_deployment(failing_probe=True) + + # Wait until the deployment's pods are running + res = test_utils.call_until_true( + self.check_pods_status_num, consts.POD_CHECK_TIMEOUT*3, + consts.POD_CHECK_SLEEP_TIME, namespace='default', label='app=demo', + num_pods=3) + self.assertTrue(res, 'Timed out waiting for pods to be running') + + # Wait until the pods are not ready + self.wait_for_status( + consts.POD_CHECK_TIMEOUT*3, consts.POD_CHECK_SLEEP_TIME, + self.check_pods_ready_num, namespace='default', label='app=demo', + num_pods=0) + + # Get current Kuryr pods restart count (for a later comparison) + controller_pods = self.get_controller_pod_names() + container_restarts_before = self.get_pod_containers_restarts( + pod_names=controller_pods, + namespace=CONF.kuryr_kubernetes.kube_system_namespace) + + # Create a service + service_name, _ = self.create_service(pod_label={"app": "demo"}, + spec_type='ClusterIP') + self.addCleanup(self.delete_service, service_name) + + # Check Kuryr pods are not restarted + self.check_controller_pod_status_for_time_period( + retry_attempts=10, + time_between_attempts=3) + + # Get current Kuryr pods restart count + container_restarts_after = self.get_pod_containers_restarts( + pod_names=controller_pods, + namespace=CONF.kuryr_kubernetes.kube_system_namespace) + + # Compare Kuryr pods restart count with previously stored data + self.assertEqual(container_restarts_before, container_restarts_after, + "Kuryr controller pod(s) were restarted during the " + "service creation, expected: %s, obtained: %s" % + (container_restarts_before, container_restarts_after))