Create test mapping for puppet-modules

As part of fuel-library-gates, adding mapping structure for puppet modules.
For each system-test as a key exist list of deployment/puppet,
deployment/Puppetfile, osnailyfactor/modular modules which it's system-test covers.
Mapping takes list of modules edit in review and register single test group - "review_fuel_library"
which mapped to system_test from mapping.

Change-Id: Ibce134668d74914bc6e3c5bdabadf6b5312dd3de
Closes-bug: #1529787

gates_tests/helpers/puppet_module_mapping.yaml

@@ -0,0 +1,109 @@
+bvt_2:
+ - apache
+ - api-proxy
+ - apt
+ - astute
+ - ceph
+ - cluster
+ - cluster-haproxy
+ - cluster-vrouter
+ - cobbler
+ - compute
+ - concat
+ - controller
+ - corosync
+ - database
+ - datacat
+ - docker
+ - fuel
+ - fuel_pkgs
+ - glance
+ - globals
+ - haproxy
+ - hiera
+ - horizon
+ - inifile
+ - keystone
+ - l23network
+ - logging
+ - mcollective
+ - memcached
+ - monit
+ - nailgun
+ - neutron
+ - nova
+ - openssl
+ - openstack
+ - openstack-controller
+ - openstack-haproxy
+ - openstack-network
+ - openstacklib
+ - pacemaker
+ - pacemaker_wrappers
+ - postgresql
+ - rabbitmq
+ - roles/controller.pp
+ - roles/compute.pp
+ - rsync
+ - rsyslog
+ - spec
+ - ssh
+ - ssl
+ - staging
+ - stdlib
+ - sysctl
+ - sysfs
+ - tftp
+ - tools
+ - tweaks
+ - vcsrepo
+ - virtual_ips
+ - xinetd
+ - vmware
+
+deploy_heat_ha:
+ - ceilometer
+ - ceilometer_ha
+ - heat
+ - heat_ha
+ - mongo
+ - mongodb
+ - roles/mongo.pp
+
+ha_neutron:
+ - cinder
+ - openstack-cinder
+ - roles/cinder.pp
+
+reduced_footprint:
+ - generate_vms
+
+ha_neutron_firewall:
+ - firewall
+
+ha_neutron_mysql_termination:
+ - galera
+ - mysql
+
+ironic_deploy_ceph:
+ - ceph
+ - ironic
+ - ironic-conductor
+ - roles/ironic-conductor.pp
+
+deploy_murano_ha_with_tun:
+ - murano
+
+deploy_ha_dns_ntp:
+ - dns
+ - ntp
+
+deploy_sahara_ha_tun:
+ - sahara
+ - sahara_templates
+
+neutron_vlan_ha:
+ - swift
+
+positive_cic_maintenance_mode:
+ - umm

gates_tests/helpers/utils.py

@@ -13,11 +13,15 @@
 #    under the License.
 
 import os
+import yaml
 
+from proboscis import register
 from proboscis.asserts import assert_equal
 from devops.helpers import helpers
 
 from fuelweb_test.helpers import checkers
+from fuelweb_test.helpers.gerrit.gerrit_info_provider import \
+    FuelLibraryModulesProvider
 from fuelweb_test.helpers.ssh_manager import SSHManager
 from fuelweb_test import logger
 from fuelweb_test import settings
@@ -486,3 +490,72 @@ def get_sha_sum(file_path):
             file_path))['stdout_str'].strip()
     logger.info('MD5 is {0}'.format(md5_sum))
     return md5_sum
+
+
+def puppet_modules_mapping(modules):
+    """
+    find fuel-qa system test which have maximum coverage for edited
+    puppet modules and register that group with "review_in_fuel_library" name
+    modules - dictionary of puppet modules edited in review
+    Example: modules = {'horizon':'fuel-library/deployment/Puppetfile'}
+    """
+
+    # open yaml with covered modules
+    with open("gates_tests/helpers/puppet_module_mapping.yaml", "r") as f:
+        mapping = yaml.load(f)
+
+    if modules and type(modules) is dict:
+        all_modules = set([j for i in mapping.values() for j in i])
+        logger.debug(
+            "List of puppet modules covered by system_tests {}".format(
+                all_modules))
+        logger.info(
+            "List of modules edited in review {}".format(modules.keys()))
+
+        # checking that module from review covered by system_test
+        for module in modules.keys():
+            if module not in all_modules:
+                logger.warning(
+                    "{}:{} module not exist or not covered by system_test"
+                    .format(module, modules[module]))
+
+        # find test group which has better coverage of modules from review
+        system_test = "bvt_2"
+        max_intersection = 0
+        if not ("ceph" in modules and set(
+                ["roles/cinder.pp", "cinder", "openstack-cinder"]) & set(
+                modules)):
+            for test in mapping:
+                test_intersection = len(
+                    set(mapping[test]).intersection(set(modules)))
+                if test_intersection > max_intersection:
+                    max_intersection = test_intersection
+                    system_test = test
+        # To completely check ceph module we can't mix ceph and cinder togeher
+        else:
+            logger.warning(
+                "We cannot check cinder and ceph together {}"
+                .format(modules))
+            system_test = "bvt_2"
+
+    else:
+        logger.warning("There no modules that changed in review "
+                       "so just run default system test")
+        system_test = "bvt_2"
+    logger.info(
+        "Puppet modules from review {}"
+        " will be checked by next system test: {}".format(
+            modules, system_test))
+
+    register(groups=['review_in_fuel_library'],
+             depends_on_groups=[system_test])
+
+
+def map_test_review_in_fuel_library(**kwargs):
+    groups = kwargs.get('run_groups', None)
+    old_groups = kwargs.get('groups', None)
+    groups.extend(old_groups or [])
+    if 'review_in_fuel_library' in groups:
+        mp = FuelLibraryModulesProvider.from_environment_vars()
+        modules = mp.get_changed_modules()
+        puppet_modules_mapping(modules)

run_system_test.py

@@ -8,6 +8,8 @@ from proboscis import register
 
 from fuelweb_test.helpers.utils import pretty_log
 
+from gates_tests.helpers.utils import map_test_review_in_fuel_library
+
 from system_test import register_system_test_cases
 from system_test import get_groups
 from system_test import define_custom_groups
@@ -193,6 +195,7 @@ def shell():
     args = cli()
     discover_import_tests(basedir, tests_directory)
     define_custom_groups()
+    map_test_review_in_fuel_library(**vars(args))
     COMMAND_MAP[args.command](**vars(args))