Fix smoke tests

Change-Id: I6c760feaa07cc282c7a05a04665b964120767ea2

plugin_test/vapor/exports.sh

@@ -11,6 +11,8 @@ export OS_FAULTS_CLOUD_DRIVER_ADDRESS=192.168.10.100
 export OS_FAULTS_CLOUD_DRIVER_KEYFILE=/home/jenkins/cloud.key
 export CLOUD_DRIVER_USERNAME=root
 
+export CONTRAIL_ROLES_DISTRIBUTION_YAML=roles_distribution_example.yaml
+
 #export OS_PROJECT_DOMAIN_NAME=default
 #export OS_USER_DOMAIN_NAME=default
 #export OS_PROJECT_NAME=admin

plugin_test/vapor/vapor/fixtures/system_services.py

@@ -10,10 +10,11 @@
 # License for the specific language governing permissions and limitations
 # under the License
 
-from kazoo.client import KazooClient
+from kazoo import client
 from vapor import settings
 import pytest
 
+
 @pytest.fixture
 def znodes_list(nodes_ips):
     hosts_list = ""
@@ -21,10 +22,10 @@ def znodes_list(nodes_ips):
         settings.ROLE_CONTRAIL_CONTROLLER]
     for name in nodes_ips:
         if name in contrail_controllers_fqdns:
-            hosts_list+="{}:{},".format(nodes_ips[name][0],
-                                        settings.ZOOKEEPER_PORT)
+            hosts_list += "{}:{},".format(nodes_ips[name][0],
+                                          settings.ZOOKEEPER_PORT)
     hosts_list = hosts_list[:-1]
-    zk = KazooClient(hosts=hosts_list)
+    zk = client.KazooClient(hosts=hosts_list)
     zk.start()
     znodes_list_ = zk.get_children("/")
     zk.stop()

plugin_test/vapor/vapor/helpers/asserts.py

@@ -0,0 +1,54 @@
+"""Assertion helpers."""
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from hamcrest import assert_that
+
+
+class ManyAssertionsErrors(AssertionError):
+    """Many assertions errors class."""
+
+    def __init__(self, exceptions=None, *args, **kwargs):
+        """Constructor."""
+        super(ManyAssertionsErrors, self).__init__(*args, **kwargs)
+        self._exceptions = exceptions or []
+
+    def __str__(self):
+        messages = (str(e) for e in self._exceptions)
+        return '\n' + '\n'.join(messages)
+
+
+class AssertsCollector(object):
+    """Assertion class to check many asserts and report all of them later."""
+
+    def __init__(self):
+        """Constructor."""
+        self._exceptions = []
+
+    def check(self, *args, **kwargs):
+        """Perform an assert check and store exception, if it was raised."""
+        try:
+            assert_that(*args, **kwargs)
+        except AssertionError as e:
+            self._exceptions.append(e)
+
+    def report(self):
+        """Raise ManyAssertionsErrors if there are any exceptions captured."""
+        if self._exceptions:
+            raise ManyAssertionsErrors(self._exceptions)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        return self.report()

plugin_test/vapor/vapor/helpers/contrail_status.py

@@ -1,64 +1,109 @@
 """`contrail-status` CLI command results parser."""
+import collections
 import re
 
-from hamcrest import assert_that, empty, has_entries  # noqa H301
+import attrdict
+from hamcrest import (empty, has_entries, contains_inanyorder,
+                      has_length)  # noqa: H301
 from six import moves
 from stepler.third_party import waiter
 
-GOOD_STATUSES = {'active', 'backup'}
+from vapor.helpers import asserts
+
+
+STATUS_ACTIVE = 'active'
+STATUS_BACKUP = 'backup'
+STATUS_INACTIVE = 'inactive (disabled on boot)'
 
 
 def parse_result(stdout_lines):
     """Function to parse result of `contrail-status`.
 
-    It returns a dict:
-        {'service': 'status', 'service', 'status', ...}
+    It yields an AttrDict:
+        {'service': 'service1',
+         'status': 'active',
+         'section':' Contrail Database'}
     """
     service_pattern = re.compile(r'(?P<name>[^\s]+)\s+(?P<status>.+)')
-    results = {}
+    section = None
     for line in stdout_lines:
         line = line.strip()
-        if not line or '==' in line:
+        if not line:
+            continue
+        if '==' in line:
+            section = line.strip(' =')
             continue
         service_result = service_pattern.search(line)
         if service_result:
             name, status = service_result.groups()
             if ':' in name:
                 name = name.split(':')[0]
-            results[name] = status
-    return results
+            yield attrdict.AttrDict(
+                service=name, status=status, section=section)
 
 
 def get_services_statuses(os_faults_steps):
     """Function to retrieve contrail services statuses on each node.
 
-    It returns a dict:
-        {'node-1.test.domain.local': {'service1': 'status',
-                                      'service2': 'status'},
-         'node-2.test.domain.local': {...},
+    It returns an AttrDict:
+        {'node-1.test.domain.local': [
+            {'name': service1',
+             'status': 'active',
+             'section': 'Contrail Database'},
+            {'name': service2',
+             'status': 'active',
+             'section': 'Contrail Database'},
+            ]
+         'node-2.test.domain.local': [...],
          ...}
     """
     cmd = 'contrail-status'
     nodes = os_faults_steps.get_nodes_by_cmd('which ' + cmd)
-    results = {}
+    results = collections.defaultdict(list)
     for node_result in os_faults_steps.execute_cmd(nodes, cmd):
         node = next(moves.filter(lambda x: x.ip == node_result.host, nodes))
-        services_statuses = parse_result(node_result.payload['stdout_lines'])
-        results[node.fqdn] = services_statuses
+        for service in parse_result(node_result.payload['stdout_lines']):
+            results[node.fqdn].append(service)
 
-    return results
+    return attrdict.AttrDict(results)
 
 
 def check_services_statuses(os_faults_steps):
     """Function to check that all contrail services are ok on nodes."""
-    broken_services = []
+    services_data = collections.defaultdict(list)
+    Status = collections.namedtuple('Status', ['node', 'status', 'section'])
     for fqdn, services in get_services_statuses(os_faults_steps).items():
-        for name, status in services.items():
-            if status not in GOOD_STATUSES:
-                err_msg = "{node}:{service} - {status}".format(
-                    node=fqdn, service=name, status=status)
-                broken_services.append(err_msg)
-    assert_that(broken_services, empty())
+        for service in services:
+            status = Status(
+                node=fqdn, status=service.status, section=service.section)
+            services_data[service.service].append(status)
+
+    with asserts.AssertsCollector() as collector:
+        for service, statuses in services_data.items():
+            err_msg = (
+                "`{}` has instances with wrong statuses").format(service)
+
+            statuses = set(statuses)
+            active = {x for x in statuses if x.status == STATUS_ACTIVE}
+            if service in ('contrail-svc-monitor', 'contrail-schema'):
+                backup = {x for x in statuses if x.status == STATUS_BACKUP}
+                broken = statuses - active - backup
+                collector.check(active,
+                                has_length(1),
+                                ('`{}` should have'
+                                 ' only one active instance').format(service))
+                collector.check(broken, empty(), err_msg)
+            elif service == 'contrail-database':
+                allowed_inactive = set()
+                for status in statuses:
+                    if (status.status == STATUS_INACTIVE and
+                            status.section == 'Contrail Database'):
+                        allowed_inactive.add(status)
+                broken = statuses - active - allowed_inactive
+                collector.check(broken, empty(), err_msg)
+            else:
+                broken = statuses - active
+                collector.check(broken, empty(), err_msg)
 
 
 def check_service_status(os_faults_steps,
@@ -69,13 +114,13 @@ def check_service_status(os_faults_steps,
     """Check that service on nodes_fqdns has expected status."""
 
     def predicate():
-        statuses = get_services_statuses(os_faults_steps)
-        return waiter.expect_that(statuses,
-                                  has_entries(**{
-                                      node: has_entries({
-                                          service: expected_status
-                                      })
-                                      for node in nodes_fqdns
-                                  }))
+        services = get_services_statuses(os_faults_steps)
+        return waiter.expect_that(
+            services,
+            has_entries(**{
+                node: contains_inanyorder(
+                    has_entries(service=service, status=expected_status))
+                for node in nodes_fqdns
+            }))
 
     waiter.wait(predicate, timeout_seconds=timeout)

plugin_test/vapor/vapor/settings.py

@@ -54,15 +54,17 @@ ROLE_CONTRAIL_COMPUTE = 'contrail-compute'
 ROLE_CONTRAIL_CONFIG = 'contrail-config'
 
 CONTRAIL_ROLES_SERVICES_MAPPING = {
+    ROLE_CONTRAIL_CONFIG: (
+        'supervisor-config',
+        'contrail-config-nodemgr',
+    ),
     ROLE_CONTRAIL_CONTROLLER: (
         'supervisor-control',
         'contrail-control',
         'contrail-control-nodemgr',
         'contrail-dns',
         'contrail-named',
-        'supervisor-config',
         'contrail-api',
-        'contrail-config-nodemgr',
         'contrail-device-manager',
         'contrail-discovery',
         'contrail-schema',
@@ -157,4 +159,3 @@ ZOOKEEPER_NODES = ["api-server",
                    "controller",
                    "schema-transformer",
                    "brokers"]
-

plugin_test/vapor/vapor/tests/common/test_base.py

@@ -174,7 +174,8 @@ def test_restart_control_service(os_faults_steps):
 
 
 @pytest.mark.usefixtures('contrail_network_cleanup')
-def test_vn_name_with_special_characters(contrail_api_client, contrail_current_project):
+def test_vn_name_with_special_characters(contrail_api_client,
+                                         contrail_current_project):
     """Validate creating virtual network with special characters in name.
 
     Steps:

plugin_test/vapor/vapor/tests/test_smoke.py

@@ -11,13 +11,15 @@
 # under the License.
 
 from hamcrest import (assert_that, has_item, has_entry, is_not, empty,
-                      has_property, has_entries,
-                      has_items,contains_inanyorder)  # noqa H301
+                      has_property, has_items,
+                      contains_inanyorder)  # noqa: H301
+import jmespath
 import pycontrail.types as types
 import pytest
 from stepler.third_party import utils
 
 from vapor.helpers import contrail_status
+from vapor.helpers import asserts
 from vapor import settings
 
 
@@ -25,15 +27,18 @@ def test_contrail_node_services_status(os_faults_steps):
     contrail_status.check_services_statuses(os_faults_steps)
 
 
-@pytest.mark.parametrize('role',
-                         settings.CONTRAIL_ROLES_DISTRIBUTION)
+@pytest.mark.parametrize('role', settings.CONTRAIL_ROLES_DISTRIBUTION)
 def test_contrail_service_distribution(os_faults_steps, role):
-    statuses = contrail_status.get_services_statuses(os_faults_steps)
-    statuses = {node: services.keys() for node, services in statuses.items()}
-    services = settings.CONTRAIL_ROLES_SERVICES_MAPPING[role]
+    """Check that contrail services are running on correct nodes."""
+    services_statuses = contrail_status.get_services_statuses(os_faults_steps)
     nodes = settings.CONTRAIL_ROLES_DISTRIBUTION[role]
-    entries = {node: has_items(*services) for node in nodes}
-    assert_that(statuses, has_entries(**entries))
+    expected_services = settings.CONTRAIL_ROLES_SERVICES_MAPPING[role]
+    with asserts.AssertsCollector() as collector:
+        for node, services in services_statuses.items():
+            if node not in nodes:
+                continue
+            services = [x.service for x in services]
+            collector.check(services, has_items(*expected_services))
 
 
 @pytest.mark.usefixtures('contrail_network_cleanup')
@@ -153,7 +158,11 @@ def test_update_network_ipam(contrail_api_client, contrail_ipam):
 
 def test_contrail_alarms_is_empty(client_contrail_analytics):
     alarms = client_contrail_analytics.get_alarms()
-    assert_that(alarms, empty())
+    query = ('*[?@.value.*.alarms[?ack!=`True`]][].'
+             '{Node: @.name, Type: @.value.*.alarms[].type}')
+    not_ack_alarms = jmespath.search(query, alarms)
+    assert_that(not_ack_alarms, empty())
+
 
 def test_zookeeper_status(znodes_list):
     expected_znodes_list = settings.ZOOKEEPER_NODES