OCP-Router - add support for OCP routes

This patch extends Kuryr's tempest plugin to verify
OCP-Route capability.

This patch includes also a generic function (named 'wait_kuryr_annotation')
for waiting (with max timeout) till specific annotation appears in object metadata.
A follow-up patch will update the current code to rely on 'wait_kuryr_annotation'
function.

Depends-On: Ia1816d2fa1468385544bb854af67c391050f3636
Change-Id: I8449ceb0735a9aea2e52d5d71a91a51dfd512c17
This commit is contained in:
Antoni Segura Puimedon 2018-08-13 15:56:39 +02:00 committed by Yossi Boaron
parent 22a7a3b09d
commit 5de75537bd
6 changed files with 223 additions and 3 deletions

View File

@ -45,3 +45,4 @@
- kuryr-kubernetes-tempest-daemon-containerized-octavia-namespace
- kuryr-kubernetes-tempest-daemon-containerized-openshift-octavia-serial
- kuryr-kubernetes-tempest-daemon-ovn
- kuryr-kubernetes-tempest-daemon-openshift-octavia-ingress

View File

@ -64,7 +64,12 @@ kubernetes_project_name = cfg.StrOpt("kubernetes_project_name",
default="k8s",
help="The OpenStack project name "
"for Kubernetes")
npwg_multi_vif_enabled = cfg.BoolOpt("npwg_multi_vif_enabled",
default=False,
help="Whether or not NPWG multi-vif "
"feature is enabled")
ocp_router_fip = cfg.StrOpt("ocp_router_fip",
default=None,
help="OCP Router floating IP")

View File

@ -52,6 +52,8 @@ class KuryrTempestPlugin(plugins.TempestPlugin):
group='kuryr_kubernetes')
conf.register_opt(project_config.npwg_multi_vif_enabled,
group='kuryr_kubernetes')
conf.register_opt(project_config.ocp_router_fip,
group='kuryr_kubernetes')
def get_opt_lists(self):
return [('service_available', [project_config.service_option]),
@ -61,6 +63,8 @@ class KuryrTempestPlugin(plugins.TempestPlugin):
project_config.containerized,
project_config.kube_system_namespace,
project_config.run_tests_serial,
project_config.npwg_multi_vif_enabled]),
project_config.npwg_multi_vif_enabled,
project_config.ocp_router_fip]),
('vif_pool', [project_config.ports_pool_batch,
project_config.lb_build_timeout])]

View File

@ -23,6 +23,7 @@ import kubernetes
from kubernetes import client as k8s_client
from kubernetes import config as k8s_config
from kubernetes.stream import stream
from openshift import dynamic
from tempest import config
from tempest.lib.common.utils import data_utils
@ -37,6 +38,8 @@ KURYR_NET_CRD_VERSION = 'v1'
KURYR_NET_CRD_PLURAL = 'kuryrnets'
K8S_ANNOTATION_PREFIX = 'openstack.org/kuryr'
K8S_ANNOTATION_LBAAS_STATE = K8S_ANNOTATION_PREFIX + '-lbaas-state'
K8S_ANNOTATION_LBAAS_RT_STATE = K8S_ANNOTATION_PREFIX + '-lbaas-route-state'
KURYR_ROUTE_DEL_VERIFY_TIMEOUT = 30
class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
@ -58,6 +61,8 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
cls.pod_fips = []
# TODO(dmellado): Config k8s client in a cleaner way
k8s_config.load_kube_config()
cls.dyn_client = dynamic.DynamicClient(
k8s_config.new_client_from_config())
@classmethod
def resource_cleanup(cls):
@ -285,6 +290,24 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
else:
raise lib_exc.NotImplemented()
@classmethod
def wait_kuryr_annotation(cls, api_version, kind, annotation,
timeout_period, name, namespace='default'):
dyn_resource = cls.dyn_client.resources.get(
api_version=api_version, kind=kind)
start = time.time()
while time.time() - start < timeout_period:
time.sleep(1)
resource = dyn_resource.get(name, namespace=namespace)
if resource.metadata.annotations is None:
continue
for resp_annotation in resource.metadata.annotations:
if annotation in resp_annotation:
return
LOG.info("Waiting till %s will appear "
"in %s/%s annotation ", annotation, kind, name)
raise lib_exc.ServerFault()
@classmethod
def wait_service_status(cls, service_ip, timeout_period):
session = requests.Session()
@ -305,7 +328,9 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
@classmethod
def create_setup_for_service_test(cls, pod_num=2, spec_type="ClusterIP",
protocol="TCP", label=None,
namespace="default", get_ip=True):
namespace="default", get_ip=True,
service_name=None):
label = label or data_utils.rand_name('kuryr-app')
for i in range(pod_num):
pod_name, pod = cls.create_pod(
@ -315,11 +340,12 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
cls.pod_num = pod_num
service_name, service_obj = cls.create_service(
pod_label=pod.metadata.labels, spec_type=spec_type,
protocol=protocol, namespace=namespace)
protocol=protocol, namespace=namespace, service_name=service_name)
if get_ip:
cls.service_ip = cls.get_service_ip(
service_name, spec_type=spec_type, namespace=namespace)
cls.verify_lbaas_endpoints_configured(service_name)
cls.service_name = service_name
cls.wait_service_status(
cls.service_ip, CONF.kuryr_kubernetes.lb_build_timeout)
@ -437,6 +463,8 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
@classmethod
def _verify_endpoints_annotation(cls, ep_name, ann_string,
poll_interval=1, namespace='default'):
LOG.info("Look for %s string in ep=%s annotation ",
ann_string, ep_name)
# wait until endpoint annotation created
while True:
time.sleep(poll_interval)
@ -445,6 +473,8 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
annotations = ep.metadata.annotations
try:
json.loads(annotations[ann_string])
LOG.info("Found %s string in ep=%s annotation ",
ann_string, ep_name)
return
except KeyError:
LOG.info("Waiting till %s will appears "
@ -487,3 +517,90 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
'Kuryr controller is not in the %s state' % status
)
retry_attempts -= 1
@classmethod
def create_route(cls, name, hostname, target_svc, namespace='default'):
route_manifest = {
'apiVersion': 'v1',
'kind': 'Route',
'metadata':
{
'name': name,
},
'spec':
{
'host': hostname,
'to':
{
'kind': 'Service',
'name': target_svc
}
}
}
v1_routes = cls.dyn_client.resources.get(api_version='v1',
kind='Route')
v1_routes.create(body=route_manifest, namespace=namespace)
LOG.info("Route=%s created, wait for kuryr-annotation", name)
cls.wait_kuryr_annotation(
api_version='route.openshift.io/v1', kind='Route',
annotation='openstack.org/kuryr-route-state',
timeout_period=90, name=name)
LOG.info("Found %s string in Route=%s annotation ",
'openstack.org/kuryr-route-state', name)
@classmethod
def verify_route_endpoints_configured(cls, ep_name, namespace='default'):
cls._verify_endpoints_annotation(
ep_name=ep_name, ann_string=K8S_ANNOTATION_LBAAS_RT_STATE,
poll_interval=3)
@classmethod
def delete_route(cls, name, namespace='default'):
v1_routes = cls.dyn_client.resources.get(api_version='v1',
kind='Route')
try:
v1_routes.delete(name, namespace=namespace)
except Exception as e:
if e.status == 404:
return
raise
# FIXME(yboaron): Use other method (instead of constant delay)
# to verify that route was deleted
time.sleep(KURYR_ROUTE_DEL_VERIFY_TIMEOUT)
def verify_route_http(self, router_fip, hostname, amount,
should_succeed=True):
LOG.info("Trying to curl the route, Router_FIP=%s, hostname=%s, "
"should_succeed=%s", router_fip, hostname, should_succeed)
def req():
if should_succeed:
# FIXME(yboaron): From some reason for route use case,
# sometimes only one of service's pods is responding to CURL
# although all members and L7 policy were created at Octavia.
# so as workaround - I added a constant delay
time.sleep(1)
resp = requests.get('http://{}'.format(router_fip),
headers={'Host': hostname})
return resp.status_code, resp.content
def pred(tester, responses):
contents = []
for resp in responses:
status_code, content = resp
if should_succeed:
contents.append(content)
else:
tester.assertEqual(requests.codes.SERVICE_UNAVAILABLE,
status_code)
if should_succeed:
unique_resps = set(contents)
tester.assertEqual(amount, len(unique_resps),
'Incorrect amount of unique backends. '
'Got {}'.format(unique_resps))
self._run_threaded_and_assert(req, pred, fn_timeout=10)

View File

@ -0,0 +1,92 @@
# Copyright 2018 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kuryr_tempest_plugin.tests.scenario import base
from oslo_log import log as logging
from tempest import config
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators
LOG = logging.getLogger(__name__)
CONF = config.CONF
FIRST_ROUTE_NAME = 'firstroute'
FIRST_ROUTE_HOST_NAME = 'www.first.com'
SECOND_ROUTE_NAME = 'secondroute'
SECOND_ROUTE_HOST_NAME = 'www.second.com'
class TestOcpRouteScenario(base.BaseKuryrScenarioTest):
@classmethod
def skip_checks(cls):
super(TestOcpRouteScenario, cls).skip_checks()
if CONF.kuryr_kubernetes.ocp_router_fip is None:
raise cls.skipException(
"OCP router fip should be specified to run this tests.")
@decorators.idempotent_id('bddf0001-1244-449d-a125-b5fdcfa1a7a9')
def test_create_route_after_service(self):
self.create_setup_for_service_test()
self.addCleanup(self.delete_route, FIRST_ROUTE_NAME)
self.create_route(FIRST_ROUTE_NAME, FIRST_ROUTE_HOST_NAME,
self.service_name)
self.verify_route_endpoints_configured(self.service_name)
self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
FIRST_ROUTE_HOST_NAME,
self.pod_num)
self.delete_route(FIRST_ROUTE_NAME)
self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
FIRST_ROUTE_HOST_NAME,
self.pod_num,
should_succeed=False)
@decorators.idempotent_id('bddf0002-1244-449d-a125-b5fdcfa1a7a9')
def test_two_routes_same_service(self):
self.create_setup_for_service_test()
self.addCleanup(self.delete_route, FIRST_ROUTE_NAME)
self.create_route(FIRST_ROUTE_NAME, FIRST_ROUTE_HOST_NAME,
self.service_name)
self.addCleanup(self.delete_route, SECOND_ROUTE_NAME)
self.create_route(SECOND_ROUTE_NAME, SECOND_ROUTE_HOST_NAME,
self.service_name)
self.verify_route_endpoints_configured(self.service_name)
self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
FIRST_ROUTE_HOST_NAME,
self.pod_num)
self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
SECOND_ROUTE_HOST_NAME,
self.pod_num)
self.delete_route(FIRST_ROUTE_NAME)
self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
SECOND_ROUTE_HOST_NAME,
self.pod_num)
self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
FIRST_ROUTE_HOST_NAME,
self.pod_num,
should_succeed=False)
@decorators.idempotent_id('bddf0003-1244-449d-a125-b5fdcfa1a7a9')
def test_create_route_before_service(self):
service_name = data_utils.rand_name(prefix='kuryr-service')
self.addCleanup(self.delete_route, FIRST_ROUTE_NAME)
self.create_route(FIRST_ROUTE_NAME, FIRST_ROUTE_HOST_NAME,
service_name)
self.create_setup_for_service_test(service_name=service_name)
self.verify_route_endpoints_configured(service_name)
self.verify_route_http(CONF.kuryr_kubernetes.ocp_router_fip,
FIRST_ROUTE_HOST_NAME,
self.pod_num)

View File

@ -11,3 +11,4 @@ tempest>=17.1.0 # Apache-2.0
testrepository>=0.0.18 # Apache-2.0/BSD
testtools>=2.2.0 # MIT
kubernetes>=5.0.0 # Apache-2.0
openshift>=0.7.0