From 418dbcb5498e44cae0113ea9d90f5e6630973df2 Mon Sep 17 00:00:00 2001 From: Artem Panchenko Date: Tue, 20 Sep 2016 14:49:49 +0300 Subject: [PATCH] Add component tests for Calico network plugin Scenario: 1. Install k8s with Calico network plugin. 2. Run netchecker-server service. 3. Run netchecker-agent daemon set. 4. Get network verification status. Check status is 'OK'. New test depends on mcp-netchecker tool. Some related to network checker stuff was moved and refactored. Change-Id: I1f1c85a943a937e11f1e61df046d3c1425dd3643 --- fuel_ccp_tests/fixtures/k8s_fixtures.py | 35 +++++++ fuel_ccp_tests/managers/k8smanager.py | 21 +++++ .../managers/underlay_ssh_manager.py | 19 ++++ fuel_ccp_tests/tests/system/test_calico.py | 65 +++++++++++++ .../tests/system/test_netchecker.py | 94 ++++--------------- 5 files changed, 160 insertions(+), 74 deletions(-) create mode 100644 fuel_ccp_tests/tests/system/test_calico.py diff --git a/fuel_ccp_tests/fixtures/k8s_fixtures.py b/fuel_ccp_tests/fixtures/k8s_fixtures.py index 897a762..2172e31 100644 --- a/fuel_ccp_tests/fixtures/k8s_fixtures.py +++ b/fuel_ccp_tests/fixtures/k8s_fixtures.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import os import pytest from fuel_ccp_tests import logger @@ -85,3 +86,37 @@ def k8scluster(revert_snapshot, request, config, pass return k8s_actions + + +@pytest.fixture(scope='class') +def check_netchecker_files(request): + files_missing = [] + for arg in request.cls.netchecker_files: + if not os.path.isfile(arg): + files_missing.append(arg) + assert len(files_missing) == 0, \ + ("The following netchecker files not found: " + "{0}!".format(', '.join(files_missing))) + + +@pytest.fixture(scope='class') +def check_netchecker_images_settings(): + settings_missing = [] + for setting in ('MCP_NETCHECKER_AGENT_IMAGE_REPO', + 'MCP_NETCHECKER_AGENT_VERSION', + 'MCP_NETCHECKER_SERVER_IMAGE_REPO', + 'MCP_NETCHECKER_SERVER_VERSION'): + if not getattr(settings, setting, None): + settings_missing.append(setting) + assert len(settings_missing) == 0, \ + ("The following environment variables are not set: " + "{0}!".format(', '.join(settings_missing))) + + +@pytest.fixture(scope='class') +def check_calico_images_settings(): + assert settings.DEFAULT_CUSTOM_YAML['kube_network_plugin'] == 'calico', \ + "Calico network plugin isn't enabled!" + if not any(settings.CALICO.values()): + LOG.warning("No custom settings are provided for Calico! " + "Defaults will be used!") diff --git a/fuel_ccp_tests/managers/k8smanager.py b/fuel_ccp_tests/managers/k8smanager.py index b98c1d4..c49e0db 100644 --- a/fuel_ccp_tests/managers/k8smanager.py +++ b/fuel_ccp_tests/managers/k8smanager.py @@ -285,6 +285,27 @@ class K8SManager(object): LOG.info("DaemonSet '{}' is created".format(ds.metadata.name)) return self.api.daemonsets.get(name=ds.metadata.name) + def check_ds_ready(self, dsname, namespace=None): + """Check if k8s DaemonSet is ready + + :param dsname: str, ds name + :return: bool + """ + ds = self.api.daemonsets.get(name=dsname, namespace=namespace) + return (ds.status.current_number_scheduled == + ds.status.desired_number_scheduled) + + def wait_ds_ready(self, dsname, namespace=None, timeout=60, interval=5): + """Wait until all pods are scheduled on nodes + + :param dsname: str, ds name + :param timeout: int + :param interval: int + """ + helpers.wait( + lambda: self.check_ds_ready(dsname, namespace=namespace), + timeout=timeout, interval=interval) + def create_objects(self, path): if isinstance(path, str): path = [path] diff --git a/fuel_ccp_tests/managers/underlay_ssh_manager.py b/fuel_ccp_tests/managers/underlay_ssh_manager.py index a6c3639..b8504c2 100644 --- a/fuel_ccp_tests/managers/underlay_ssh_manager.py +++ b/fuel_ccp_tests/managers/underlay_ssh_manager.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +import random + from devops.helpers import helpers from devops.helpers import ssh_client from paramiko import rsakey @@ -309,3 +311,20 @@ class UnderlaySSHManager(object): command=cmd, verbose=verbose, timeout=timeout, error_info=error_info, expected=expected, raise_on_err=raise_on_err) + + def dir_upload(self, host, source, destination): + """Upload local directory content to remote host + + :param host: str, remote node name + :param source: str, local directory path + :param destination: str, local directory path + """ + with self.remote(node_name=host) as remote: + remote.upload(source, destination) + + def get_random_node(self): + """Get random node name + + :return: str, name of node + """ + return random.choice(self.node_names()) diff --git a/fuel_ccp_tests/tests/system/test_calico.py b/fuel_ccp_tests/tests/system/test_calico.py new file mode 100644 index 0000000..cd105a1 --- /dev/null +++ b/fuel_ccp_tests/tests/system/test_calico.py @@ -0,0 +1,65 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pytest + +import base_test +import test_netchecker + +from fuel_ccp_tests import logger + +LOG = logger.logger + + +@pytest.mark.usefixtures("check_netchecker_files") +@pytest.mark.usefixtures("check_netchecker_images_settings") +@pytest.mark.usefixtures("check_calico_images_settings") +@pytest.mark.component +class TestFuelCCPCalico(base_test.SystemBaseTest, + test_netchecker.TestFuelCCPNetCheckerMixin): + """Test class for Calico network plugin in k8s""" + + @pytest.mark.fail_snapshot + @pytest.mark.snapshot_needed + def test_k8s_netchecker_calico(self, underlay, k8scluster, config, + show_step): + """Test for deploying k8s environment with Calico plugin and check + network connectivity between pods + + Scenario: + 1. Install k8s with Calico network plugin. + 2. Run netchecker-server service. + 3. Run netchecker-agent daemon set. + 4. Get network verification status. Check status is 'OK'. + + Duration: 3000 seconds + """ + + # STEP #1 + show_step(1) + k8sclient = k8scluster.api + assert k8sclient.nodes.list() is not None, "Can not get nodes list" + + # STEP #2 + show_step(2) + self.start_netchecker_server(k8s=k8scluster) + self.wait_netchecker_running(config.k8s.kube_host, timeout=240) + + # STEP #3 + show_step(3) + self.start_netchecker_agent(underlay, k8scluster) + + # STEP #4 + show_step(4) + self.wait_check_network(config.k8s.kube_host, works=True) diff --git a/fuel_ccp_tests/tests/system/test_netchecker.py b/fuel_ccp_tests/tests/system/test_netchecker.py index 294820a..6b18088 100644 --- a/fuel_ccp_tests/tests/system/test_netchecker.py +++ b/fuel_ccp_tests/tests/system/test_netchecker.py @@ -14,7 +14,6 @@ import os import pytest -import random import requests import yaml @@ -29,31 +28,6 @@ from fuel_ccp_tests import settings LOG = logger.logger -@pytest.fixture(scope='class') -def check_netchecker_files(request): - files_missing = [] - for arg in request.cls.netchecker_files: - if not os.path.isfile(arg): - files_missing.append(arg) - assert len(files_missing) == 0, \ - ("The following netchecker files not found: " - "{0}!".format(', '.join(files_missing))) - - -@pytest.fixture(scope='class') -def check_netchecker_images(): - settings_missing = [] - for setting in ('MCP_NETCHECKER_AGENT_IMAGE_REPO', - 'MCP_NETCHECKER_AGENT_VERSION', - 'MCP_NETCHECKER_SERVER_IMAGE_REPO', - 'MCP_NETCHECKER_SERVER_VERSION'): - if not getattr(settings, setting, None): - settings_missing.append(setting) - assert len(settings_missing) == 0, \ - ("The following environment variables are not set: " - "{0}!".format(', '.join(settings_missing))) - - class TestFuelCCPNetCheckerMixin: pod_yaml_file = os.path.join( settings.NETCHECKER_SERVER_DIR, @@ -65,31 +39,6 @@ class TestFuelCCPNetCheckerMixin: settings.NETCHECKER_AGENT_DIR, 'netchecker-agent.yaml') netchecker_files = (pod_yaml_file, svc_yaml_file, ds_yaml_file) - -@pytest.mark.usefixtures("check_netchecker_files") -@pytest.mark.usefixtures("check_netchecker_images") -@pytest.mark.system -class TestFuelCCPNetChecker(base_test.SystemBaseTest, - TestFuelCCPNetCheckerMixin): - """Test class for network connectivity verification in k8s""" - - @staticmethod - def dir_upload(underlay, host, source, destination): - with underlay.remote(node_name=host) as remote: - remote.upload(source, destination) - - @staticmethod - def get_ds_status(k8s, dsname): - ds = k8s.api.daemonsets.get(name=dsname) - return (ds.status.current_number_scheduled == - ds.status.desired_number_scheduled) - - @staticmethod - def wait_ds_running(k8s, dsname, timeout=60, interval=5): - helpers.wait( - lambda: TestFuelCCPNetChecker.get_ds_status(k8s, dsname), - timeout=timeout, interval=interval) - def start_netchecker_server(self, k8s): with open(self.pod_yaml_file) as pod_conf: for pod_spec in yaml.load_all(pod_conf): @@ -142,9 +91,7 @@ class TestFuelCCPNetChecker(base_test.SystemBaseTest, settings.MCP_NETCHECKER_AGENT_IMAGE_REPO, settings.MCP_NETCHECKER_AGENT_VERSION) k8s.check_ds_create(body=daemon_set_spec) - TestFuelCCPNetChecker.wait_ds_running( - k8s, - dsname=daemon_set_spec['metadata']['name']) + k8s.wait_ds_ready(dsname=daemon_set_spec['metadata']['name']) @staticmethod def get_netchecker_status(kube_host_ip, netchecker_pod_port=31081): @@ -171,29 +118,29 @@ class TestFuelCCPNetChecker(base_test.SystemBaseTest, timeout=timeout, interval=interval) @staticmethod - def get_random_slave(underlay): - slave_nodes = [n for n in underlay.node_names() if n != 'master'] - if not slave_nodes: - return None - random.shuffle(slave_nodes) - return slave_nodes.pop() - - @staticmethod - def block_traffic_on_slave(underlay, slave_node): + def calico_block_traffic_on_node(underlay, target_node): LOG.info('Blocked traffic to the network checker service from ' - 'containers on node "{}".'.format(slave_node)) + 'containers on node "{}".'.format(target_node)) underlay.sudo_check_call( 'calicoctl profile calico-k8s-network rule add ' '--at=1 outbound deny tcp to ports 8081', - node_name=slave_node) + node_name=target_node) @staticmethod - def unblock_traffic_on_slave(underlay, slave_node): + def calico_unblock_traffic_on_node(underlay, target_node): LOG.info('Unblocked traffic to the network checker service from ' - 'containers on node "{}".'.format(slave_node)) + 'containers on node "{}".'.format(target_node)) underlay.sudo_check_call( 'calicoctl profile calico-k8s-network rule remove outbound --at=1', - node_name=slave_node) + node_name=target_node) + + +@pytest.mark.usefixtures("check_netchecker_files") +@pytest.mark.usefixtures("check_netchecker_images_settings") +@pytest.mark.component +class TestFuelCCPNetChecker(base_test.SystemBaseTest, + TestFuelCCPNetCheckerMixin): + """Test class for network connectivity verification in k8s""" @pytest.mark.fail_snapshot @pytest.mark.snapshot_needed @@ -208,10 +155,10 @@ class TestFuelCCPNetChecker(base_test.SystemBaseTest, 2. Run netchecker-server service 3. Run netchecker-agent daemon set 4. Get network verification status. Check status is 'OK' - 5. Randomly choose some slave, login to it via SSH, add blocking + 5. Randomly choose some k8s node, login to it via SSH, add blocking rule to the calico policy. Restart network checker server 6. Get network verification status, Check status is 'FAIL' - 7. Recover calico profile state on the slave + 7. Recover calico profile state on the node 8. Get network verification status. Check status is 'OK' Duration: 300 seconds @@ -232,14 +179,13 @@ class TestFuelCCPNetChecker(base_test.SystemBaseTest, self.start_netchecker_agent(underlay, k8scluster) # STEP #4 - # currently agents need some time to start reporting to the server show_step(4) self.wait_check_network(config.k8s.kube_host, works=True) # STEP #5 show_step(5) - target_slave = self.get_random_slave(underlay) - self.block_traffic_on_slave(underlay, target_slave) + target_node = underlay.get_random_node() + self.calico_block_traffic_on_node(underlay, target_node) # STEP #6 show_step(6) @@ -247,7 +193,7 @@ class TestFuelCCPNetChecker(base_test.SystemBaseTest, # STEP #7 show_step(7) - self.unblock_traffic_on_slave(underlay, target_slave) + self.calico_unblock_traffic_on_node(underlay, target_node) # STEP #8 show_step(8)