Merge "Add devops_template for each test in fuel_library and puppet-openstack mapping"

2016-08-30 05:13:41 +00:00 · 2016-08-30 05:13:41 +00:00 · 3c878da14c
parent 3b3552e496 8d3b5e3de9
commit 3c878da14c
5 changed files with 415 additions and 100 deletions
--- a/fuelweb_test/tests/base_test_case.py
+++ b/fuelweb_test/tests/base_test_case.py
@ -12,9 +12,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.

+import os
 import time

 from devops.helpers.helpers import wait
+from devops.helpers.templates import yaml_template_load

 from proboscis import TestProgram
 from proboscis import SkipTest
@ -83,7 +85,13 @@ class TestBasic(object):
    @property
    def env(self):
        if self.__env is None:
-            self.__env = EnvironmentModel(self._devops_config)
+            # hack before we find proper solution for loading devops templates
+            external_config = os.environ.get("DEVOPS_SETTINGS_TEMPLATE", None)
+            if external_config:
+                config = yaml_template_load(external_config)
+            else:
+                config = self._devops_config
+            self.__env = EnvironmentModel(config)
        return self.__env

    @property
--- a/gates_tests/devops_templates/default.yaml
+++ b/gates_tests/devops_templates/default.yaml
@ -0,0 +1,235 @@
+---
+aliases:
+
+  dynamic_address_pool:
+   - &pool_default !os_env POOL_DEFAULT, 10.109.0.0/16:24
+
+  default_interface_model:
+   - &interface_model !os_env INTERFACE_MODEL, e1000
+
+template:
+  devops_settings:
+    env_name: !os_env ENV_NAME
+
+    address_pools:
+    # Network pools used by the environment
+
+      fuelweb_admin-pool01:
+        net: *pool_default
+        params:
+          vlan_start: 0
+
+          # Reserved IP addresses.
+          # - 'l2_network_device' will be used as an IP address for creating
+          #   libvirt network with assigned IP on it, if specified.
+          # - 'gateway' should be used in fuel-qa framework to get the gateway
+          #   of the address pool. Can be different from libvirt IP address
+          #   in case if a network already have another gateway.
+          # - all other IP addresses specified in the template can be used
+          #   for different purposes in different test cases.
+          ip_reserved:
+
+            # Address of the gateway in the address pool fuelweb_admin-pool01.
+            # If string - than is used 'as is', as a real IP address
+            # If numeric - than assumed as a relative address
+            #              from the start of the address pool.
+            gateway: +1
+
+            # Libvirt bridge will get this IP address
+            l2_network_device: +1
+
+          # IP ranges, reserved in the address pool.
+          # - 'default' IP range stands for 'default' nodegroup in Fuel.
+          # - any other IP range names such as 'floating' can be specified to
+          #   provide a special IP ranges to configure cluster settings.
+          #   For special test cases, several IP ranges can be configured here
+          #   for different networks. Names of these ranges should be
+          #   agreed for template and test cases.
+          ip_ranges:
+
+            # Relative or real IP addresses
+            default: [+2, -2]     # admin IP range for 'default' nodegroup name
+
+      public-pool01:
+        net: *pool_default
+        params:
+          vlan_start: 0
+          ip_reserved:
+            gateway: +1
+            l2_network_device: +1  # l2_network_device will get this IP address
+          ip_ranges:
+            default: [+2, +127]  # public IP range for 'default' nodegroup name
+            floating: [+128, -2]
+
+      storage-pool01:
+        net: *pool_default
+        params:
+          vlan_start: 101
+
+      management-pool01:
+        net: *pool_default
+        params:
+          vlan_start: 102
+
+      private-pool01:
+        net: *pool_default
+        params:
+          vlan_start: 960
+          vlan_end: 1000
+
+    groups:
+     - name: rack-01
+       driver:
+         name: devops.driver.libvirt
+         params:
+           connection_string: !os_env CONNECTION_STRING, qemu:///system
+           storage_pool_name: !os_env STORAGE_POOL_NAME, default
+           stp: True
+           hpet: False
+           use_host_cpu: !os_env DRIVER_USE_HOST_CPU, true
+
+       network_pools:  # Address pools for OpenStack networks.
+         # Actual names should be used for keys
+         # (the same as in Nailgun, for example)
+
+         fuelweb_admin: fuelweb_admin-pool01
+         public: public-pool01
+         storage: storage-pool01
+         management: management-pool01
+         private: private-pool01
+
+       l2_network_devices:  # Libvirt bridges. It is *NOT* Nailgun networks
+         admin:
+           address_pool: fuelweb_admin-pool01
+           dhcp: false
+           forward:
+             mode: nat
+
+         public:
+           address_pool: public-pool01
+           dhcp: false
+           forward:
+             mode: nat
+
+         storage:
+           address_pool: storage-pool01
+           dhcp: false
+
+         management:
+           address_pool: management-pool01
+           dhcp: false
+
+         private:
+           address_pool: private-pool01
+           dhcp: false
+
+       nodes:
+        - name: admin        # Custom name of VM for Fuel admin node
+          role: fuel_master  # Fixed role for Fuel master node properties
+          params:
+            vcpu: !os_env ADMIN_NODE_CPU, 2
+            memory: !os_env ADMIN_NODE_MEMORY, 3072
+            boot:
+              - hd
+              - cdrom  # for boot from usb - without 'cdrom'
+            volumes:
+             - name: system
+               capacity: !os_env ADMIN_NODE_VOLUME_SIZE, 75
+               format: qcow2
+             - name: iso
+               source_image: !os_env ISO_PATH    # if 'source_image' set, then volume capacity is calculated from it's size
+               format: raw
+               device: cdrom   # for boot from usb - 'disk'
+               bus: ide        # for boot from usb - 'usb'
+            interfaces:
+             - label: eth0
+               l2_network_device: admin    # Libvirt bridge name. It is *NOT* a Nailgun network
+               interface_model: *interface_model
+            network_config:
+              eth0:
+                networks:
+                 - fuelweb_admin
+
+          # Slave nodes
+
+        - name: slave-01
+          role: fuel_slave
+          params:  &rack-01-slave-node-params
+            vcpu: !os_env SLAVE_NODE_CPU, 2
+            memory: !os_env SLAVE_NODE_MEMORY, 3072
+            boot:
+              - network
+              - hd
+            volumes:
+             - name: system
+               capacity: !os_env NODE_VOLUME_SIZE, 50
+               format: qcow2
+             - name: cinder
+               capacity: !os_env NODE_VOLUME_SIZE, 50
+               format: qcow2
+             - name: swift
+               capacity: !os_env NODE_VOLUME_SIZE, 50
+               format: qcow2
+
+            # List of node interfaces
+            interfaces:
+             - label: eth0
+               l2_network_device: admin      # Libvirt bridge name. It is *NOT* Nailgun networks
+               interface_model: *interface_model
+             - label: eth1
+               l2_network_device: public
+               interface_model: *interface_model
+             - label: eth2
+               l2_network_device: storage
+               interface_model: *interface_model
+             - label: eth3
+               l2_network_device: management
+               interface_model: *interface_model
+             - label: eth4
+               l2_network_device: private
+               interface_model: *interface_model
+
+            # How Nailgun/OpenStack networks should assigned for interfaces
+            network_config:
+              eth0:
+                networks:
+                 - fuelweb_admin  # Nailgun/OpenStack network name
+              eth1:
+                networks:
+                 - public
+              eth2:
+                networks:
+                 - storage
+              eth3:
+                networks:
+                 - management
+              eth4:
+                networks:
+                 - private
+
+
+        - name: slave-02
+          role: fuel_slave
+          params: *rack-01-slave-node-params
+        - name: slave-03
+          role: fuel_slave
+          params: *rack-01-slave-node-params
+        - name: slave-04
+          role: fuel_slave
+          params: *rack-01-slave-node-params
+        - name: slave-05
+          role: fuel_slave
+          params: *rack-01-slave-node-params
+        - name: slave-06
+          role: fuel_slave
+          params: *rack-01-slave-node-params
+        - name: slave-07
+          role: fuel_slave
+          params: *rack-01-slave-node-params
+        - name: slave-08
+          role: fuel_slave
+          params: *rack-01-slave-node-params
+        - name: slave-09
+          role: fuel_slave
+          params: *rack-01-slave-node-params
+
--- a/gates_tests/helpers/fuel_library_modules_mapping.yaml
+++ b/gates_tests/helpers/fuel_library_modules_mapping.yaml
@ -1,103 +1,109 @@
 bvt_2:
- - apache
- - apache_api_proxy
- - apache_mpm
- - api-proxy
- - api_proxy.conf
- - apt
- - astute
- - atop
- - atop_retention
- - auth_file
- - credentials_file
- - ceph
- - ceph_nova_compute
- - cluster
- - cluster_haproxy
- - cluster-haproxy
- - cluster_vrouter
- - cluster-vrouter
- - cobbler
- - compute
- - concat
- - controller
- - corosync
- - database
- - datacat
- - deployment_groups
- - dnsmasq
- - dnsmasq.conf
- - docker
- - fuel
- - fuel_pkgs
- - glance
- - globals
- - globals_yaml
- - haproxy
- - hiera
- - horizon
- - hosts
- - inifile
- - install_ssh_keys
- - keystone
- - l23network
- - logging
- - mon
- - mcollective
- - memcached
- - monit
- - nailgun
- - neutron
- - netconfig
- - nova
- - openrc.erb
- - openssl
- - openstack
- - openstack-controller
- - openstack_controller
- - openstack-haproxy
- - openstack_haproxy
- - openstack-network
- - openstack_network
- - openstacklib
- - pacemaker
- - pacemaker_wrappers
- - package_pins
- - postgresql
- - puppet_pull
- - puppet-pull.sh
- - rabbitmq
- - roles/controller
- - roles/compute
- - roles/enable_compute
- - roles/tasks
- - rsync
- - rsyslog
- - spec
- - ssh
- - ssl
- - staging
- - stdlib
- - sysctl
- - sysfs
- - test_compute
- - test_controller
- - tftp
- - tools
- - tweaks
- - vcsrepo
- - virtual_ips
- - xinetd
- - vmware
- - wait_for_backend
- - wait_for_keystone_backends
- - wait_for_glance_backends
+ devops_settings_template: devops_templates/default.yaml
+ modules:
+   - apache
+   - apache_api_proxy
+   - apache_mpm
+   - api-proxy
+   - api_proxy.conf
+   - apt
+   - astute
+   - atop
+   - atop_retention
+   - auth_file
+   - credentials_file
+   - ceph
+   - ceph_nova_compute
+   - cluster
+   - cluster_haproxy
+   - cluster-haproxy
+   - cluster_vrouter
+   - cluster-vrouter
+   - cobbler
+   - compute
+   - concat
+   - controller
+   - corosync
+   - database
+   - datacat
+   - deployment_groups
+   - dnsmasq
+   - dnsmasq.conf
+   - docker
+   - fuel
+   - fuel_pkgs
+   - glance
+   - globals
+   - globals_yaml
+   - haproxy
+   - hiera
+   - horizon
+   - hosts
+   - inifile
+   - install_ssh_keys
+   - keystone
+   - l23network
+   - logging
+   - mon
+   - mcollective
+   - memcached
+   - monit
+   - nailgun
+   - neutron
+   - netconfig
+   - nova
+   - openrc.erb
+   - openssl
+   - openstack
+   - openstack-controller
+   - openstack_controller
+   - openstack-haproxy
+   - openstack_haproxy
+   - openstack-network
+   - openstack_network
+   - openstacklib
+   - pacemaker
+   - pacemaker_wrappers
+   - package_pins
+   - postgresql
+   - puppet_pull
+   - puppet-pull.sh
+   - rabbitmq
+   - roles/controller
+   - roles/compute
+   - roles/enable_compute
+   - roles/tasks
+   - rsync
+   - rsyslog
+   - spec
+   - ssh
+   - ssl
+   - staging
+   - stdlib
+   - sysctl
+   - sysfs
+   - test_compute
+   - test_controller
+   - tftp
+   - tools
+   - tweaks
+   - vcsrepo
+   - virtual_ips
+   - xinetd
+   - vmware
+   - wait_for_backend
+   - wait_for_keystone_backends
+   - wait_for_glance_backends


 deploy_ha_controller_neutron_example:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - plugins

 deploy_heat_ha:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - aodh
 - ceilometer
 - ceilometer_ha
@ -108,6 +114,8 @@ deploy_heat_ha:
 - roles/mongo

 ha_neutron:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - cinder
 - openstack-cinder
 - roles/cinder
@ -115,13 +123,19 @@ ha_neutron:
 - openstack_cinder

 reduced_footprint:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - generate_vms
 - vm_libvirt

 ha_neutron_firewall:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - firewall

 ha_neutron_mysql_termination:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - galera
 - mysql
 - mysql.access.cnf
@ -130,6 +144,8 @@ ha_neutron_mysql_termination:
 - mysql_user_access

 ironic_deploy_ceph:
+ devops_settings_template: devops_templates/ironic_template.yaml
+ modules:
 - ceph
 - ironic
 - ironic-conductor
@ -137,9 +153,13 @@ ironic_deploy_ceph:
 - roles/ironic_conductor

 deploy_murano_ha_with_tun:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - murano

 deploy_ha_dns_ntp:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - dns
 - ntp
 - resolvconf
@ -147,22 +167,34 @@ deploy_ha_dns_ntp:
 - resolv.dnsmasq.conf

 deploy_sahara_ha_tun:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - sahara
 - sahara_templates

 neutron_vlan_ha:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - swift
 - proxy

 positive_cic_maintenance_mode:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - umm

 deploy_ha_cgroup:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - cgroups

 basic_env_for_hugepages:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - allocated_hugepages

 test_logrotate:
+ devops_settings_template: devops_templates/default.yaml
+ modules:
 - apache2.logrotate
 - apache2.prerotate
--- a/gates_tests/helpers/openstack_puppet_projects_mapping.yaml
+++ b/gates_tests/helpers/openstack_puppet_projects_mapping.yaml
@ -1,4 +1,6 @@
 bvt_2:
+ devops_settings_template: devops_templates/default.yaml
+ projects:
 - openstack/puppet-ceph
 - openstack/puppet-glance
 - openstack/puppet-horizon
@ -10,19 +12,29 @@ bvt_2:
 - openstack/puppet-oslo

 neutron_vlan_ha:
+ devops_settings_template: devops_templates/default.yaml
+ projects:
 - openstack/puppet-cinder
 - openstack/puppet-swift

 deploy_sahara_ha_tun:
+ devops_settings_template: devops_templates/default.yaml
+ projects:
 - openstack/puppet-sahara

 deploy_murano_ha_with_tun:
+ devops_settings_template: devops_templates/default.yaml
+ projects:
 - openstack/puppet-murano

 ironic_deploy_ceph:
+ devops_settings_template: devops_templates/ironic_template.yaml
+ projects:
 - openstack/puppet-ironic

 deploy_heat_ha:
+ devops_settings_template: devops_templates/default.yaml
+ projects:
 - openstack/puppet-aodh
 - openstack/puppet-ceilometer
 - openstack/puppet-heat
--- a/gates_tests/helpers/utils.py
+++ b/gates_tests/helpers/utils.py
@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.

+import itertools
 import os
 import yaml

@ -19,7 +20,6 @@ from proboscis import register
 from proboscis.asserts import assert_equal
 from devops.helpers import helpers

-
 from fuelweb_test.helpers.fuel_actions import BaseActions
 from fuelweb_test.helpers.gerrit.gerrit_info_provider import \
    FuelLibraryModulesProvider
@ -304,7 +304,9 @@ def fuel_library_modules_mapping(modules):
        mapping = yaml.load(f)

    if modules and isinstance(modules, dict):
-        all_modules = set([j for i in mapping.values() for j in i])
+        all_modules = set(list(itertools.chain.from_iterable(
+            [mapping[test_group]['modules'] for test_group in mapping])))
+
        logger.debug(
            "List of puppet modules covered by system_tests {}".format(
                all_modules))
@ -313,7 +315,7 @@ def fuel_library_modules_mapping(modules):

        # checking that module from review covered by system_test
        for module in modules.keys():
-            if module.split('.')[0] not in all_modules:
+            if module not in all_modules:
                logger.warning(
                    "{}:{} module not exist or not covered by system_test"
                    .format(module, modules[module]))
@ -326,10 +328,23 @@ def fuel_library_modules_mapping(modules):
                set(modules)):
            for test in mapping:
                test_intersection = len(
-                    set(mapping[test]).intersection(set(modules)))
+                    set(mapping[test]['modules']).intersection(set(modules)))
                if test_intersection > max_intersection:
                    max_intersection = test_intersection
                    system_test = test
+
+            devops_template = mapping[system_test]['devops_settings_template']
+
+            import gates_tests
+
+            path_to_template = os.path.join(
+                os.path.dirname(os.path.abspath(gates_tests.__file__)),
+                devops_template)
+
+            logger.debug("devops template is {}".format(path_to_template))
+
+            os.environ['DEVOPS_SETTINGS_TEMPLATE'] = path_to_template
+
        # To completely check ceph module we can't mix ceph and cinder togeher
        else:
            logger.warning(
@ -365,7 +380,8 @@ def openstack_puppet_project_mapping(project):
            "r") as f:
        mapping = yaml.load(f)

-        all_projects = set([j for i in mapping.values() for j in i])
+        all_projects = set(list(itertools.chain.from_iterable(
+            [mapping[test_group]['modules'] for test_group in mapping])))
        logger.debug(
            "List of openstack/puppet-projects "
            "covered by system_tests {}".format(
@ -382,9 +398,21 @@ def openstack_puppet_project_mapping(project):
        # find test group which cover project edited in review
        system_test = "bvt_2"
        for test in mapping:
-            if project in mapping[test]:
+            if project in mapping[test]['projects']:
                system_test = test
                break
+
+        devops_template = mapping[system_test]['devops_settings_template']
+
+        import gates_tests
+
+        path_to_template = os.path.join(
+            os.path.dirname(os.path.abspath(gates_tests.__file__)),
+            devops_template)
+
+        logger.debug("devops template is {}".format(path_to_template))
+
+        os.environ['DEVOPS_SETTINGS_TEMPLATE'] = path_to_template
        logger.info(
            "Edited project in review - '{}'"
            " will be checked by next system test: {}".format(