From 76684464c09cb3f710fb382a123c1263aad453a1 Mon Sep 17 00:00:00 2001 From: Georgy Dyuldin Date: Wed, 22 Feb 2017 09:50:23 +0300 Subject: [PATCH] Add test_agent_cleanup_with_control_node_stop Change-Id: Idcd7a67e83d3351c395c24569dd19140a65312ad --- plugin_test/vapor/vapor/fixtures/nodes.py | 17 ++++ plugin_test/vapor/vapor/fixtures/skip.py | 11 ++- .../vapor/vapor/helpers/analytic_steps.py | 22 +++++ plugin_test/vapor/vapor/helpers/asserts.py | 44 ++++++--- plugin_test/vapor/vapor/settings.py | 6 ++ .../vapor/vapor/tests/test_analytics.py | 4 +- .../vapor/vapor/tests/test_destructive.py | 91 +++++++++++++++++++ plugin_test/vapor/vapor/tests/test_smoke.py | 4 +- 8 files changed, 181 insertions(+), 18 deletions(-) diff --git a/plugin_test/vapor/vapor/fixtures/nodes.py b/plugin_test/vapor/vapor/fixtures/nodes.py index 03ba3384b..54a5eabf6 100644 --- a/plugin_test/vapor/vapor/fixtures/nodes.py +++ b/plugin_test/vapor/vapor/fixtures/nodes.py @@ -24,3 +24,20 @@ def computes(os_faults_steps): fqdns = settings.CONTRAIL_ROLES_DISTRIBUTION[ settings.ROLE_CONTRAIL_COMPUTE] return os_faults_steps.get_nodes(fqdns) + + +@pytest.fixture +def stop_service(os_faults_steps): + """Callable fixture to stop service on nodes.""" + stopped = [] + + def _stop_service(nodes, service): + cmd = "service {} stop".format(service) + os_faults_steps.execute_cmd(nodes, cmd) + stopped.append([nodes, service]) + + yield _stop_service + + for nodes, service in reversed(stopped): + cmd = "service {} start".format(service) + os_faults_steps.execute_cmd(nodes, cmd) diff --git a/plugin_test/vapor/vapor/fixtures/skip.py b/plugin_test/vapor/vapor/fixtures/skip.py index 6c5aa9980..e5828fb85 100644 --- a/plugin_test/vapor/vapor/fixtures/skip.py +++ b/plugin_test/vapor/vapor/fixtures/skip.py @@ -27,8 +27,8 @@ class Predicates(skip.Predicates): def vrouter_headless_mode(self): """Define whether vrouter headless mode enabled.""" os_faults_steps = self._get_fixture('os_faults_steps') - fqdns = settings.CONTRAIL_ROLES_DISTRIBUTION[settings. - ROLE_CONTRAIL_COMPUTE] + fqdns = settings.CONTRAIL_ROLES_DISTRIBUTION[ + settings.ROLE_CONTRAIL_COMPUTE] actual_nodes = os_faults_steps.get_nodes_by_cmd( settings.VROUTER_HEADLESS_MODE_CMD) return set(actual_nodes.get_fqdns()) == set(fqdns) @@ -41,6 +41,13 @@ class Predicates(skip.Predicates): sriov_device_mappings = sriov.get_sriov_device_mapping(agent_steps) return len(sriov_device_mappings) > 0 + @property + @_store_call + def contrail_control_nodes_count(self): + """Return contrail control nodes count.""" + return len(settings.CONTRAIL_ROLES_DISTRIBUTION[ + settings.ROLE_CONTRAIL_CONTROLLER]) + @pytest.fixture def predicates(request): diff --git a/plugin_test/vapor/vapor/helpers/analytic_steps.py b/plugin_test/vapor/vapor/helpers/analytic_steps.py index ba1776c3f..96cec1527 100644 --- a/plugin_test/vapor/vapor/helpers/analytic_steps.py +++ b/plugin_test/vapor/vapor/helpers/analytic_steps.py @@ -65,6 +65,28 @@ def get_vna_xmpp_connection_status(session, ip, port): return result +def get_vna_vm_list(session, ip, port): + """Return vna vm list.""" + response = session.get( + 'http://{ip}:{port}/Snh_VmListReq?uuid='.format( + ip=ip, port=port)) + response.raise_for_status() + tree = ET.fromstring(response.content) + avn = tree.findall('.//list/VmSandeshData/uuid') + return [x.text for x in avn] + + +def wait_vna_vm_list(session, ip, port, matcher, timeout): + """Wait until received vm_list will satisfy the macher.""" + + def predicate(): + list_of_vm = get_vna_vm_list(session, ip, port) + + return waiter.expect_that(list_of_vm, matcher) + + waiter.wait(predicate, timeout_seconds=timeout) + + def get_processes_with_wrong_state(ops, module_id, expected_state='Functional'): """Return ops with wrong state for module_id.""" diff --git a/plugin_test/vapor/vapor/helpers/asserts.py b/plugin_test/vapor/vapor/helpers/asserts.py index 2440586f5..f0a6641e2 100644 --- a/plugin_test/vapor/vapor/helpers/asserts.py +++ b/plugin_test/vapor/vapor/helpers/asserts.py @@ -75,37 +75,57 @@ class BaseSequencesMatcher(BaseMatcher): return not self._get_wrong_items(item) -class IsSubsetOf(BaseSequencesMatcher): +class SubsetOf(BaseSequencesMatcher): - def _get_wrong_items(self, item): - return self._get_extra_items(item, self.sequence) + def _get_wrong_items(self, other): + return self._get_extra_items(other, self.sequence) def describe_to(self, description): description.append_text('a subset of ').append_text(self.sequence) def describe_mismatch(self, item, mismatch_description): - super(IsSubsetOf, self).describe_mismatch(item, mismatch_description) + super(SubsetOf, self).describe_mismatch(item, mismatch_description) mismatch_description.append_text(' with unexpected items ') \ .append_description_of(self._get_wrong_items(item)) -class IsSupersetOf(BaseSequencesMatcher): +class SupersetOf(BaseSequencesMatcher): - def _get_wrong_items(self, item): - return self._get_extra_items(self.sequence, item) + def _get_wrong_items(self, other): + return self._get_extra_items(self.sequence, other) def describe_to(self, description): description.append_text('a superset of ').append_text(self.sequence) def describe_mismatch(self, item, mismatch_description): - super(IsSupersetOf, self).describe_mismatch(item, mismatch_description) + super(SupersetOf, self).describe_mismatch(item, mismatch_description) mismatch_description.append_text(' without expected items ') \ .append_description_of(self._get_wrong_items(item)) -def is_subset_of(sequence): - return IsSubsetOf(sequence) +class IntersectsWith(BaseSequencesMatcher): + + def _matches(self, other): + for item in self.sequence: + if item in other: + return True + return False + + def describe_to(self, description): + description.append_text( + 'a sequence which intersect with ').append_text(self.sequence) -def is_superset_of(sequence): - return IsSupersetOf(sequence) +def subset_of(other): + """Matches if sequence is subset of `other`.""" + return SubsetOf(other) + + +def superset_of(other): + """Matches if sequence is superset of `other`.""" + return SupersetOf(other) + + +def intersects_with(other): + """Matches if sequence has intersections with `other`.""" + return IntersectsWith(other) diff --git a/plugin_test/vapor/vapor/settings.py b/plugin_test/vapor/vapor/settings.py index cf3b78693..400775b4c 100644 --- a/plugin_test/vapor/vapor/settings.py +++ b/plugin_test/vapor/vapor/settings.py @@ -47,6 +47,12 @@ PING_SUCCESS_TIMEOUT = 60 * 2 # Security group apply timeout SECURITY_GROUP_APPLY_TIMEOUT = 20 +# Time to wait for contrail agent cleanup after stopping control node +CONTRAIL_AGENT_CLEANUP_TIMEOUT = 20 * 60 + +# Time to wait for contrail agent vna vm list to contain server uuid +CONTRAIL_AGENT_VNA_VM_LIST_TIMEOUT = 3 * 60 + ROLE_CONTRAIL_CONTROLLER = 'contrail-controller' ROLE_CONTRAIL_ANALYTICS = 'contrail-analytics' ROLE_CONTRAIL_DB = 'contrail-db' diff --git a/plugin_test/vapor/vapor/tests/test_analytics.py b/plugin_test/vapor/vapor/tests/test_analytics.py index d02705dee..1dc840cc8 100644 --- a/plugin_test/vapor/vapor/tests/test_analytics.py +++ b/plugin_test/vapor/vapor/tests/test_analytics.py @@ -7,7 +7,7 @@ from six.moves import filter from vapor.helpers import analytic_steps from vapor.helpers import asserts -from vapor.helpers.asserts import is_superset_of +from vapor.helpers.asserts import superset_of from vapor import settings @@ -242,7 +242,7 @@ def test_uve_module_states(client_contrail_analytics, os_faults_steps, role, 'NodeStatus.process_info[].process_name', data) process_list = [x.split(':')[0] for x in process_list] collector.check(process_list, - is_superset_of(expected_process_list), + superset_of(expected_process_list), 'Processes are absent on {}'.format(node)) wrong_processes = analytic_steps.get_process_info_with_wrong_state( data) diff --git a/plugin_test/vapor/vapor/tests/test_destructive.py b/plugin_test/vapor/vapor/tests/test_destructive.py index 9edb985c9..7049a5b56 100644 --- a/plugin_test/vapor/vapor/tests/test_destructive.py +++ b/plugin_test/vapor/vapor/tests/test_destructive.py @@ -15,6 +15,8 @@ import pytest from stepler import config as stepler_config from vapor import settings +from vapor.helpers.asserts import intersects_with +from vapor.helpers import analytic_steps pytestmark = pytest.mark.destructive @@ -139,3 +141,92 @@ def test_connectivity_after_contrail_services_restart( # Verify network creation network = request.getfixturevalue('contrail_network') assert_that(network.uuid, is_not(None)) + + +def test_agent_cleanup_with_control_node_stop( + session, nodes_ips, contrail_services_http_introspect_ports, + cirros_image, flavor, security_group, network, subnet, public_network, + create_floating_ip, stop_service, port_steps, server_steps, + os_faults_steps): + """Stop all the control node and verify the cleanup process in agent. + + Steps: + #. Create network, subnet + #. Create 2 servers + #. Add floating IP addresses to servers + #. Check ping to servers' floating ips + #. Get servers' ids from contrail_vrouter_agent + #. Get contrail control nodes connected to this contrail_vrouter_agent + #. Stop `contrail-control` service on found nodes + #. Check that servers' ids list from contrail_vrouter_agent is not + contain servers ids after some time + #. Start `contrail-control` service on found nodes + #. Check that servers' ids list from contrail_vrouter_agent is contain + servers ids after some time + #. Check ping to servers' floating ips + """ + # Create servers + servers = server_steps.create_servers( + count=2, + image=cirros_image, + flavor=flavor, + security_groups=[security_group], + networks=[network], + username=stepler_config.CIRROS_USERNAME, + password=stepler_config.CIRROS_PASSWORD) + + # Create Floating IP + for server in servers: + server_port = port_steps.get_port( + device_owner=stepler_config.PORT_DEVICE_OWNER_SERVER, + device_id=server.id) + + floating_ip = create_floating_ip(public_network, port=server_port) + server_steps.check_server_ip( + server, + floating_ip['floating_ip_address'], + timeout=settings.FLOATING_IP_BIND_TIMEOUT) + + for server in servers: + server_steps.check_ping_to_server_floating( + server, timeout=stepler_config.PING_CALL_TIMEOUT) + + servers_ids = [s.id for s in servers] + compute_fqdn = getattr(servers[0], + settings.SERVER_ATTR_HYPERVISOR_HOSTNAME) + agent_ip = nodes_ips[compute_fqdn][0] + agent_port = contrail_services_http_introspect_ports[ + 'contrail-vrouter-agent']['port'] + + analytic_steps.wait_vna_vm_list( + session, agent_ip, agent_port, + intersects_with(servers_ids), + settings.CONTRAIL_AGENT_VNA_VM_LIST_TIMEOUT) + + # Collecting control nodes + controllers_fqdns = [] + for entry in analytic_steps.get_vna_xmpp_connection_status( + session, agent_ip, agent_port): + ip = entry['controller_ip'] + fqdn = next(fqnd for fqnd, ips in nodes_ips.items() if ip in ips) + controllers_fqdns.append(fqdn) + controller_nodes = os_faults_steps.get_nodes(fqdns=controllers_fqdns) + + # Stop contrail-control service + stop_service(controller_nodes, 'contrail-control') + + analytic_steps.wait_vna_vm_list(session, agent_ip, agent_port, + is_not(intersects_with(servers_ids)), + settings.CONTRAIL_AGENT_CLEANUP_TIMEOUT) + + os_faults_steps.execute_cmd(controller_nodes, + 'service contrail-control start') + + analytic_steps.wait_vna_vm_list( + session, agent_ip, agent_port, + intersects_with(servers_ids), + settings.CONTRAIL_AGENT_VNA_VM_LIST_TIMEOUT) + + for server in servers: + server_steps.check_ping_to_server_floating( + server, timeout=stepler_config.PING_CALL_TIMEOUT) diff --git a/plugin_test/vapor/vapor/tests/test_smoke.py b/plugin_test/vapor/vapor/tests/test_smoke.py index 1a95d0a42..b3ccafa91 100644 --- a/plugin_test/vapor/vapor/tests/test_smoke.py +++ b/plugin_test/vapor/vapor/tests/test_smoke.py @@ -19,7 +19,7 @@ from stepler.third_party import utils from vapor.helpers import contrail_status from vapor.helpers import asserts -from vapor.helpers.asserts import is_superset_of +from vapor.helpers.asserts import superset_of from vapor import settings @@ -38,7 +38,7 @@ def test_contrail_service_distribution(os_faults_steps, role): if node not in nodes: continue services = [x.service for x in services] - collector.check(services, is_superset_of(expected_services)) + collector.check(services, superset_of(expected_services)) def test_ifmap_service(os_faults_steps):