Merge "Test restart of CNI and Controller kuryr pods"

kuryr_tempest_plugin/config.py

@@ -43,3 +43,18 @@ service_tests_enabled = cfg.BoolOpt("service_tests_enabled",
                                     default=True,
                                     help="Whether or not service tests "
                                          "will be running")
+
+containerized = cfg.BoolOpt("containerized",
+                            default=False,
+                            help="Whether or not kuryr-controller and "
+                                 "kuryr-cni are containerized")
+
+kube_system_namespace = cfg.StrOpt("kube_system_namespace",
+                                   default="kube-system",
+                                   help="Namespace where kuryr-controllers "
+                                        "and kuryr-cnis run")
+
+run_tests_serial = cfg.BoolOpt("run_tests_serial",
+                               default=False,
+                               help="Whether or not test run serially or "
+                                    "in parallel")

kuryr_tempest_plugin/plugin.py

@@ -37,16 +37,25 @@ class KuryrTempestPlugin(plugins.TempestPlugin):
         conf.register_opt(project_config.port_pool_enabled,
                           group='kuryr_kubernetes')
         conf.register_opt(project_config.lb_build_timeout,
+                          group='kuryr_kubernetes'),
+        conf.register_opt(project_config.containerized,
                           group='kuryr_kubernetes')
         conf.register_opt(project_config.namespace_enabled,
                           group='kuryr_kubernetes')
         conf.register_opt(project_config.service_tests_enabled,
                           group='kuryr_kubernetes')
+        conf.register_opt(project_config.kube_system_namespace,
+                          group='kuryr_kubernetes')
+        conf.register_opt(project_config.run_tests_serial,
+                          group='kuryr_kubernetes')
 
     def get_opt_lists(self):
         return [('service_available', [project_config.service_option]),
                 ('kuryr_kubernetes', [project_config.port_pool_enabled,
                                       project_config.namespace_enabled,
-                                      project_config.service_tests_enabled]),
+                                      project_config.service_tests_enabled,
+                                      project_config.containerized,
+                                      project_config.kube_system_namespace,
+                                      project_config.run_tests_serial]),
                 ('vif_pool', [project_config.ports_pool_batch,
                               project_config.lb_build_timeout])]

kuryr_tempest_plugin/tests/scenario/base.py

@@ -271,3 +271,8 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
         return cls.k8s_client.CustomObjectsApi().get_cluster_custom_object(
             group=KURYR_NET_CRD_GROUP, version=KURYR_NET_CRD_VERSION,
             plural=KURYR_NET_CRD_PLURAL, name=name)
+
+    def get_pod_name_list(self, namespace="default"):
+        return [pod.metadata.name for pod in
+                self.k8s_client.CoreV1Api().list_namespaced_pod(
+                    namespace=namespace).items]

kuryr_tempest_plugin/tests/scenario/test_kuryr_restart.py

@@ -0,0 +1,77 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import testtools
+import time
+
+from tempest import config
+from tempest.lib import decorators
+from tempest.lib import exceptions as lib_exc
+
+
+from kuryr_tempest_plugin.tests.scenario import base
+
+CONF = config.CONF
+
+
+class TestKuryrRestartScenario(base.BaseKuryrScenarioTest):
+
+    @testtools.skipUnless(
+        CONF.kuryr_kubernetes.containerized and
+        CONF.kuryr_kubernetes.run_tests_serial,
+        "CNI and controller should be containerized and this test should run "
+        "on gate, configured to run sequentially")
+    @decorators.idempotent_id('bddf5441-1244-449d-a125-b5fdcfb1a1a7')
+    def test_kuryr_pod_delete(self):
+        # find kuryr CNI and controller pods, delete them one by one and create
+        #  a regular pod just after removal
+        kube_system_pods = self.get_pod_name_list(
+            namespace=CONF.kuryr_kubernetes.kube_system_namespace)
+        for kuryr_pod_name in kube_system_pods:
+            if kuryr_pod_name.startswith('kuryr'):
+                self.delete_pod(
+                    pod_name=kuryr_pod_name,
+                    body={"kind": "DeleteOptions",
+                          "apiVersion": "v1",
+                          "gracePeriodSeconds": 0},
+                    namespace=CONF.kuryr_kubernetes.kube_system_namespace)
+
+                # make sure the kuryr pod was deleted
+                pod_delete_retries = 30
+                while self.get_pod_status(
+                        kuryr_pod_name,
+                        namespace=CONF.kuryr_kubernetes.kube_system_namespace):
+                    time.sleep(1)
+                    pod_delete_retries -= 1
+                    if pod_delete_retries == 0:
+                        raise lib_exc.TimeoutException()
+
+                # Create a new pod while kuryr CNI or kuryr controller Pods are
+                # not in the running state.
+                # Check once for controller kuryr pod and once for CNI pod
+                pod_name, pod = self.create_pod()
+                self.addCleanup(self.delete_pod, pod_name)
+                pod_fip = self.assign_fip_to_pod(pod_name)
+                if not self.ping_ip_address(
+                        pod_fip['floatingip']['floating_ip_address']):
+                    raise lib_exc.CommandFailed()
+
+        # Check that both kuryr-pods are up and running
+        # The newly created pods are running because create_pod is written
+        # that way. Will be refactored in another patch
+        for new_kuryr_pod in self.get_pod_name_list(
+                namespace=CONF.kuryr_kubernetes.kube_system_namespace):
+            self.assertEqual("Running", self.get_pod_status(
+                new_kuryr_pod,
+                namespace=CONF.kuryr_kubernetes.kube_system_namespace))