openstack
/
kuryr-tempest-plugin
Code Issues Proposed changes

Add test_service_with_not_ready_endpoints test 

This test case automates the issue described in
BZ https://bugzilla.redhat.com/show_bug.cgi?id=1980957.

The new test case:
 - creates a deployment with a failing readiness probe so all
   the pods are not ready
 - creates a service for the deployment (where the endpoints
   are not ready)
 - it checks Kuryr pods are not restarted

Change-Id: Id19f85625901b35900f9441aace8fe8938cb1d68
This commit is contained in:
Jon Uriarte 2021-12-21 08:51:53 +00:00
parent d9a93bcba9
commit 2ffdd2f475
3 changed files with 89 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
kuryr_tempest_plugin/tests/scenario/base.py
View File

@ -284,13 +284,21 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
return False
@classmethod
def get_pods_ready_num(cls, namespace="default",
label="", num_pods=1):
def check_pods_ready_num(cls, namespace="default",
label="", num_pods=1):
pods = cls.get_pod_name_list(namespace=namespace,
label_selector=label)
ready_pods = sum([cls.get_readiness_state(p) for p in pods])
return (num_pods == ready_pods)
@classmethod
def check_pods_status_num(cls, namespace="default", label="", num_pods=1,
status="Running"):
pods = cls.get_pod_name_list(namespace=namespace,
label_selector=label)
status_pods = sum([cls.get_pod_status(p) == status for p in pods])
return (num_pods == status_pods)
@classmethod
def get_pod_readiness(cls, pod_name, namespace="default"):
LOG.info("Checking if pod {} is ready".format(pod_name))
@ -406,7 +414,7 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
def create_deployment(self, deployment_name=None, api_version="apps/v1",
kind="Deployment", namespace="default",
labels={"app": "demo"}):
labels={"app": "demo"}, failing_probe=False):
api_instance = kubernetes.client.AppsV1Api()
if not deployment_name:
deployment_name = data_utils.rand_name(prefix='kuryr-deployment')
@ -418,6 +426,12 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
{"image": "quay.io/kuryr/demo",
"name": 'demo',
"ports": [{"containerPort": 8080}]}]}}
if failing_probe:
for container in template["spec"]["containers"]:
container["readinessProbe"] = {"httpGet": {"path": "/healthz",
"port": 8089},
"initialDelaySeconds": 2,
"timeoutSeconds": 1}
spec = k8s_client.V1DeploymentSpec(
replicas=3,
selector={"matchLabels": {"app": "demo"}}, template=template)
@ -440,7 +454,7 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
# NOTE(juriarte): Wait timeout increased from 180 to 300 in order to
# give the pods time to transition to ready status in the gates (and
# slow environments).
self.wait_for_status(300, 15, self.get_pods_ready_num,
self.wait_for_status(300, 15, self.check_pods_ready_num,
namespace=namespace, label=label,
num_pods=replicas)
@ -1622,3 +1636,14 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
raise lib_exc.TimeoutException("Expected num of members is %s but"
" actual is %s" % (expected_members,
num_members))
@classmethod
def get_pod_containers_restarts(cls, pod_names, namespace='default'):
containers = {}
for pod_name in pod_names:
containers[pod_name] = {}
pod = cls.k8s_client.CoreV1Api().read_namespaced_pod(pod_name,
namespace)
for container in pod.status.container_statuses:
containers[pod_name][container.name] = container.restart_count
return containers

2
kuryr_tempest_plugin/tests/scenario/consts.py
View File

@ -19,5 +19,7 @@ POD_AFFINITY = {'requiredDuringSchedulingIgnoredDuringExecution': [
'topologyKey': 'kubernetes.io/hostname'}]}
TIME_TO_APPLY_SGS = 30
POD_STATUS_RETRIES = 240
POD_CHECK_TIMEOUT = 240
POD_CHECK_SLEEP_TIME = 5
NS_TIMEOUT = 600
REPETITIONS_PER_BACKEND = 10

58
kuryr_tempest_plugin/tests/scenario/test_service.py
View File

@ -18,10 +18,12 @@ import kubernetes
from oslo_log import log as logging
from tempest import config
from tempest.lib.common.utils import data_utils
from tempest.lib.common.utils import test_utils
from tempest.lib import decorators
from tempest.lib import exceptions as lib_exc
from kuryr_tempest_plugin.tests.scenario import base
from kuryr_tempest_plugin.tests.scenario import consts
LOG = logging.getLogger(__name__)
CONF = config.CONF
@ -340,3 +342,59 @@ class TestLoadBalancerReconciliationScenario(base.BaseKuryrScenarioTest):
# if there is a connectivity now, that means the LoadBalancer
# is reconciled
self.check_service_internal_connectivity(service_name=service_name)
class TestServiceWithNotReadyEndpoints(base.BaseKuryrScenarioTest):
@classmethod
def skip_checks(cls):
super(TestServiceWithNotReadyEndpoints, cls).skip_checks()
if not CONF.kuryr_kubernetes.service_tests_enabled:
raise cls.skipException("Service tests are not enabled")
if not CONF.kuryr_kubernetes.containerized:
raise cls.skipException("Only runs on containerized setups")
@decorators.idempotent_id('bddf5441-1244-450d-a125-b5fdcfa1a7b0')
def test_service_with_not_ready_endpoints(self):
# Create a deployment with a failing probe
deployment_name, _ = self.create_deployment(failing_probe=True)
# Wait until the deployment's pods are running
res = test_utils.call_until_true(
self.check_pods_status_num, consts.POD_CHECK_TIMEOUT*3,
consts.POD_CHECK_SLEEP_TIME, namespace='default', label='app=demo',
num_pods=3)
self.assertTrue(res, 'Timed out waiting for pods to be running')
# Wait until the pods are not ready
self.wait_for_status(
consts.POD_CHECK_TIMEOUT*3, consts.POD_CHECK_SLEEP_TIME,
self.check_pods_ready_num, namespace='default', label='app=demo',
num_pods=0)
# Get current Kuryr pods restart count (for a later comparison)
controller_pods = self.get_controller_pod_names()
container_restarts_before = self.get_pod_containers_restarts(
pod_names=controller_pods,
namespace=CONF.kuryr_kubernetes.kube_system_namespace)
# Create a service
service_name, _ = self.create_service(pod_label={"app": "demo"},
spec_type='ClusterIP')
self.addCleanup(self.delete_service, service_name)
# Check Kuryr pods are not restarted
self.check_controller_pod_status_for_time_period(
retry_attempts=10,
time_between_attempts=3)
# Get current Kuryr pods restart count
container_restarts_after = self.get_pod_containers_restarts(
pod_names=controller_pods,
namespace=CONF.kuryr_kubernetes.kube_system_namespace)
# Compare Kuryr pods restart count with previously stored data
self.assertEqual(container_restarts_before, container_restarts_after,
"Kuryr controller pod(s) were restarted during the "
"service creation, expected: %s, obtained: %s" %
(container_restarts_before, container_restarts_after))