Merge "Add kuryr-controller restart test to ports pool"

kuryr_tempest_plugin/tests/scenario/base.py

@@ -99,6 +99,17 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
             body=body,
             namespace=namespace)
 
+    @classmethod
+    def wait_for_pod_status(cls, pod_name, namespace="default",
+                            pod_status=None, retries=30):
+        while pod_status != cls.get_pod_status(
+                pod_name,
+                namespace=CONF.kuryr_kubernetes.kube_system_namespace):
+            time.sleep(1)
+            retries -= 1
+            if retries == 0:
+                raise lib_exc.TimeoutException()
+
     @classmethod
     def get_pod_ip(cls, pod_name, namespace="default"):
         pod_list = cls.k8s_client.CoreV1Api().list_namespaced_pod(
@@ -115,6 +126,20 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
             if pod.metadata.name == pod_name:
                 return pod.status.phase
 
+    @classmethod
+    def get_pod_readiness(cls, pod_name, namespace="default",
+                          container_name=None):
+        pod_list = cls.k8s_client.CoreV1Api().list_namespaced_pod(
+            namespace=namespace)
+        for pod in pod_list.items:
+            if pod.metadata.name == pod_name:
+                for container in pod.status.containerStatuses:
+                    if container_name:
+                        if container.name == container_name:
+                            return container.ready
+                    else:
+                        return container.ready
+
     def get_pod_port(self, pod_name, namespace="default"):
         pod = self.k8s_client.CoreV1Api().read_namespaced_pod_status(
             namespace=namespace, name=pod_name)

kuryr_tempest_plugin/tests/scenario/test_port_pool.py

@@ -12,9 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import time
+
 from oslo_log import log as logging
 from tempest import config
 from tempest.lib import decorators
+from tempest.lib import exceptions as lib_exc
 
 from kuryr_tempest_plugin.tests.scenario import base
 
@@ -56,3 +59,53 @@ class TestPortPoolScenario(base.BaseKuryrScenarioTest):
         updated2_port_list_num = len(
             self.os_admin.ports_client.list_ports()['ports'])
         self.assertEqual(updated_port_list_num, updated2_port_list_num)
+
+        # to test the reload of the pools, we will also test the restart of the
+        # kuryr-controller
+        kube_system_pods = self.get_pod_name_list(
+            namespace=CONF.kuryr_kubernetes.kube_system_namespace)
+        for kuryr_pod_name in kube_system_pods:
+            if kuryr_pod_name.startswith('kuryr-controller'):
+                self.delete_pod(
+                    pod_name=kuryr_pod_name,
+                    body={"kind": "DeleteOptions",
+                          "apiVersion": "v1",
+                          "gracePeriodSeconds": 0},
+                    namespace=CONF.kuryr_kubernetes.kube_system_namespace)
+
+                # make sure the kuryr pod was deleted
+                self.wait_for_pod_status(
+                    kuryr_pod_name,
+                    namespace=CONF.kuryr_kubernetes.kube_system_namespace)
+
+        # Check that new kuryr-controller is up and running
+        kube_system_pods = self.get_pod_name_list(
+            namespace=CONF.kuryr_kubernetes.kube_system_namespace)
+        for kube_system_pod in kube_system_pods:
+            if kube_system_pod.startswith('kuryr-controller'):
+                self.wait_for_pod_status(
+                    kube_system_pod,
+                    namespace=CONF.kuryr_kubernetes.kube_system_namespace,
+                    pod_status='Running',
+                    retries=120)
+
+                # Wait until kuryr-controller pools are reloaded, i.e.,
+                # kuryr-controller is ready
+                pod_readiness_retries = 30
+                while not self.get_pod_readiness(
+                        kube_system_pod,
+                        namespace=CONF.kuryr_kubernetes.kube_system_namespace,
+                        container_name='controller'):
+                    time.sleep(1)
+                    pod_readiness_retries -= 1
+                    if pod_readiness_retries == 0:
+                        raise lib_exc.TimeoutException()
+
+        # create additional pod
+        pod_name, pod = self.create_pod()
+        self.addCleanup(self.delete_pod, pod_name, pod)
+
+        # the port pool should stay the same
+        updated3_port_list_num = len(
+            self.os_admin.ports_client.list_ports()['ports'])
+        self.assertEqual(updated_port_list_num, updated3_port_list_num)