diff --git a/.zuul.yaml b/.zuul.yaml index 500509fd..cbcf4b66 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -45,3 +45,4 @@ - kuryr-kubernetes-tempest-daemon-containerized-octavia-namespace - kuryr-kubernetes-tempest-daemon-containerized-openshift-octavia-serial - kuryr-kubernetes-tempest-daemon-ovn + - kuryr-kubernetes-tempest-daemon-openshift-octavia-ingress diff --git a/kuryr_tempest_plugin/config.py b/kuryr_tempest_plugin/config.py index 0ad668d2..16e1070d 100644 --- a/kuryr_tempest_plugin/config.py +++ b/kuryr_tempest_plugin/config.py @@ -64,7 +64,12 @@ kubernetes_project_name = cfg.StrOpt("kubernetes_project_name", default="k8s", help="The OpenStack project name " "for Kubernetes") + npwg_multi_vif_enabled = cfg.BoolOpt("npwg_multi_vif_enabled", default=False, help="Whether or not NPWG multi-vif " "feature is enabled") + +ocp_router_fip = cfg.StrOpt("ocp_router_fip", + default=None, + help="OCP Router floating IP") diff --git a/kuryr_tempest_plugin/plugin.py b/kuryr_tempest_plugin/plugin.py index 4915b981..731fd7ad 100644 --- a/kuryr_tempest_plugin/plugin.py +++ b/kuryr_tempest_plugin/plugin.py @@ -52,6 +52,8 @@ class KuryrTempestPlugin(plugins.TempestPlugin): group='kuryr_kubernetes') conf.register_opt(project_config.npwg_multi_vif_enabled, group='kuryr_kubernetes') + conf.register_opt(project_config.ocp_router_fip, + group='kuryr_kubernetes') def get_opt_lists(self): return [('service_available', [project_config.service_option]), @@ -61,6 +63,8 @@ class KuryrTempestPlugin(plugins.TempestPlugin): project_config.containerized, project_config.kube_system_namespace, project_config.run_tests_serial, - project_config.npwg_multi_vif_enabled]), + project_config.npwg_multi_vif_enabled, + project_config.ocp_router_fip]), + ('vif_pool', [project_config.ports_pool_batch, project_config.lb_build_timeout])] diff --git a/kuryr_tempest_plugin/tests/scenario/base.py b/kuryr_tempest_plugin/tests/scenario/base.py index 85a4a42a..e05e52c1 100644 --- a/kuryr_tempest_plugin/tests/scenario/base.py +++ b/kuryr_tempest_plugin/tests/scenario/base.py @@ -23,6 +23,7 @@ import kubernetes from kubernetes import client as k8s_client from kubernetes import config as k8s_config from kubernetes.stream import stream +from openshift import dynamic from tempest import config from tempest.lib.common.utils import data_utils @@ -37,6 +38,8 @@ KURYR_NET_CRD_VERSION = 'v1' KURYR_NET_CRD_PLURAL = 'kuryrnets' K8S_ANNOTATION_PREFIX = 'openstack.org/kuryr' K8S_ANNOTATION_LBAAS_STATE = K8S_ANNOTATION_PREFIX + '-lbaas-state' +K8S_ANNOTATION_LBAAS_RT_STATE = K8S_ANNOTATION_PREFIX + '-lbaas-route-state' +KURYR_ROUTE_DEL_VERIFY_TIMEOUT = 30 class BaseKuryrScenarioTest(manager.NetworkScenarioTest): @@ -58,6 +61,8 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest): cls.pod_fips = [] # TODO(dmellado): Config k8s client in a cleaner way k8s_config.load_kube_config() + cls.dyn_client = dynamic.DynamicClient( + k8s_config.new_client_from_config()) @classmethod def resource_cleanup(cls): @@ -285,6 +290,24 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest): else: raise lib_exc.NotImplemented() + @classmethod + def wait_kuryr_annotation(cls, api_version, kind, annotation, + timeout_period, name, namespace='default'): + dyn_resource = cls.dyn_client.resources.get( + api_version=api_version, kind=kind) + start = time.time() + while time.time() - start < timeout_period: + time.sleep(1) + resource = dyn_resource.get(name, namespace=namespace) + if resource.metadata.annotations is None: + continue + for resp_annotation in resource.metadata.annotations: + if annotation in resp_annotation: + return + LOG.info("Waiting till %s will appear " + "in %s/%s annotation ", annotation, kind, name) + raise lib_exc.ServerFault() + @classmethod def wait_service_status(cls, service_ip, timeout_period): session = requests.Session() @@ -305,7 +328,9 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest): @classmethod def create_setup_for_service_test(cls, pod_num=2, spec_type="ClusterIP", protocol="TCP", label=None, - namespace="default", get_ip=True): + namespace="default", get_ip=True, + service_name=None): + label = label or data_utils.rand_name('kuryr-app') for i in range(pod_num): pod_name, pod = cls.create_pod( @@ -315,11 +340,12 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest): cls.pod_num = pod_num service_name, service_obj = cls.create_service( pod_label=pod.metadata.labels, spec_type=spec_type, - protocol=protocol, namespace=namespace) + protocol=protocol, namespace=namespace, service_name=service_name) if get_ip: cls.service_ip = cls.get_service_ip( service_name, spec_type=spec_type, namespace=namespace) cls.verify_lbaas_endpoints_configured(service_name) + cls.service_name = service_name cls.wait_service_status( cls.service_ip, CONF.kuryr_kubernetes.lb_build_timeout) @@ -437,6 +463,8 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest): @classmethod def _verify_endpoints_annotation(cls, ep_name, ann_string, poll_interval=1, namespace='default'): + LOG.info("Look for %s string in ep=%s annotation ", + ann_string, ep_name) # wait until endpoint annotation created while True: time.sleep(poll_interval) @@ -445,6 +473,8 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest): annotations = ep.metadata.annotations try: json.loads(annotations[ann_string]) + LOG.info("Found %s string in ep=%s annotation ", + ann_string, ep_name) return except KeyError: LOG.info("Waiting till %s will appears " @@ -487,3 +517,90 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest): 'Kuryr controller is not in the %s state' % status ) retry_attempts -= 1 + + @classmethod + def create_route(cls, name, hostname, target_svc, namespace='default'): + route_manifest = { + 'apiVersion': 'v1', + 'kind': 'Route', + 'metadata': + { + 'name': name, + }, + 'spec': + { + 'host': hostname, + 'to': + { + 'kind': 'Service', + 'name': target_svc + } + } + } + + v1_routes = cls.dyn_client.resources.get(api_version='v1', + kind='Route') + v1_routes.create(body=route_manifest, namespace=namespace) + + LOG.info("Route=%s created, wait for kuryr-annotation", name) + cls.wait_kuryr_annotation( + api_version='route.openshift.io/v1', kind='Route', + annotation='openstack.org/kuryr-route-state', + timeout_period=90, name=name) + LOG.info("Found %s string in Route=%s annotation ", + 'openstack.org/kuryr-route-state', name) + + @classmethod + def verify_route_endpoints_configured(cls, ep_name, namespace='default'): + cls._verify_endpoints_annotation( + ep_name=ep_name, ann_string=K8S_ANNOTATION_LBAAS_RT_STATE, + poll_interval=3) + + @classmethod + def delete_route(cls, name, namespace='default'): + v1_routes = cls.dyn_client.resources.get(api_version='v1', + kind='Route') + try: + v1_routes.delete(name, namespace=namespace) + except Exception as e: + if e.status == 404: + return + raise + + # FIXME(yboaron): Use other method (instead of constant delay) + # to verify that route was deleted + time.sleep(KURYR_ROUTE_DEL_VERIFY_TIMEOUT) + + def verify_route_http(self, router_fip, hostname, amount, + should_succeed=True): + LOG.info("Trying to curl the route, Router_FIP=%s, hostname=%s, " + "should_succeed=%s", router_fip, hostname, should_succeed) + + def req(): + if should_succeed: + # FIXME(yboaron): From some reason for route use case, + # sometimes only one of service's pods is responding to CURL + # although all members and L7 policy were created at Octavia. + # so as workaround - I added a constant delay + time.sleep(1) + + resp = requests.get('http://{}'.format(router_fip), + headers={'Host': hostname}) + return resp.status_code, resp.content + + def pred(tester, responses): + contents = [] + for resp in responses: + status_code, content = resp + if should_succeed: + contents.append(content) + else: + tester.assertEqual(requests.codes.SERVICE_UNAVAILABLE, + status_code) + if should_succeed: + unique_resps = set(contents) + tester.assertEqual(amount, len(unique_resps), + 'Incorrect amount of unique backends. ' + 'Got {}'.format(unique_resps)) + + self._run_threaded_and_assert(req, pred, fn_timeout=10) diff --git a/kuryr_tempest_plugin/tests/scenario/test_ocp_route.py b/kuryr_tempest_plugin/tests/scenario/test_ocp_route.py new file mode 100644 index 00000000..c218be84 --- /dev/null +++ b/kuryr_tempest_plugin/tests/scenario/test_ocp_route.py @@ -0,0 +1,92 @@ +# Copyright 2018 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from kuryr_tempest_plugin.tests.scenario import base +from oslo_log import log as logging +from tempest import config +from tempest.lib.common.utils import data_utils +from tempest.lib import decorators + + +LOG = logging.getLogger(__name__) +CONF = config.CONF + +FIRST_ROUTE_NAME = 'firstroute' +FIRST_ROUTE_HOST_NAME = 'www.first.com' +SECOND_ROUTE_NAME = 'secondroute' +SECOND_ROUTE_HOST_NAME = 'www.second.com' + + +class TestOcpRouteScenario(base.BaseKuryrScenarioTest): + + @classmethod + def skip_checks(cls): + super(TestOcpRouteScenario, cls).skip_checks() + if CONF.kuryr_kubernetes.ocp_router_fip is None: + raise cls.skipException( + "OCP router fip should be specified to run this tests.") + + @decorators.idempotent_id('bddf0001-1244-449d-a125-b5fdcfa1a7a9') + def test_create_route_after_service(self): + self.create_setup_for_service_test() + self.addCleanup(self.delete_route, FIRST_ROUTE_NAME) + self.create_route(FIRST_ROUTE_NAME, FIRST_ROUTE_HOST_NAME, + self.service_name) + + self.verify_route_endpoints_configured(self.service_name) + self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip, + FIRST_ROUTE_HOST_NAME, + self.pod_num) + self.delete_route(FIRST_ROUTE_NAME) + self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip, + FIRST_ROUTE_HOST_NAME, + self.pod_num, + should_succeed=False) + + @decorators.idempotent_id('bddf0002-1244-449d-a125-b5fdcfa1a7a9') + def test_two_routes_same_service(self): + self.create_setup_for_service_test() + self.addCleanup(self.delete_route, FIRST_ROUTE_NAME) + self.create_route(FIRST_ROUTE_NAME, FIRST_ROUTE_HOST_NAME, + self.service_name) + self.addCleanup(self.delete_route, SECOND_ROUTE_NAME) + self.create_route(SECOND_ROUTE_NAME, SECOND_ROUTE_HOST_NAME, + self.service_name) + self.verify_route_endpoints_configured(self.service_name) + + self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip, + FIRST_ROUTE_HOST_NAME, + self.pod_num) + self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip, + SECOND_ROUTE_HOST_NAME, + self.pod_num) + self.delete_route(FIRST_ROUTE_NAME) + self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip, + SECOND_ROUTE_HOST_NAME, + self.pod_num) + self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip, + FIRST_ROUTE_HOST_NAME, + self.pod_num, + should_succeed=False) + + @decorators.idempotent_id('bddf0003-1244-449d-a125-b5fdcfa1a7a9') + def test_create_route_before_service(self): + service_name = data_utils.rand_name(prefix='kuryr-service') + self.addCleanup(self.delete_route, FIRST_ROUTE_NAME) + self.create_route(FIRST_ROUTE_NAME, FIRST_ROUTE_HOST_NAME, + service_name) + self.create_setup_for_service_test(service_name=service_name) + self.verify_route_endpoints_configured(service_name) + self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip, + FIRST_ROUTE_HOST_NAME, + self.pod_num) diff --git a/requirements.txt b/requirements.txt index 92247288..5acbb16f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ tempest>=17.1.0 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD testtools>=2.2.0 # MIT kubernetes>=5.0.0 # Apache-2.0 +openshift>=0.7.0