Create a test matrix role

Modify test-matrix.py so that is can be used as an ansible module.
Add a role in d-g that runs test matrix for the given branch,
role and features file.

Change-Id: Ie36ba0cd7cfcd450b75000a76a64d856f2a83eba

.zuul.yaml

@@ -3,3 +3,11 @@
     nodes:
       - name: primary
         label: ubuntu-xenial
+
+- project:
+    name: openstack-infra/devstack-gate
+    check:
+      jobs:
+        - devstack:
+            files:
+              - ^roles/

devstack-vm-gate-wrap.sh

@@ -149,7 +149,7 @@ export BASE=/opt/stack
 export ZUUL_URL=${ZUUL_URL:-http://zuul.openstack.org/p}
 
 # The feature matrix to select devstack-gate components
-export DEVSTACK_GATE_FEATURE_MATRIX=${DEVSTACK_GATE_FEATURE_MATRIX:-features.yaml}
+export DEVSTACK_GATE_FEATURE_MATRIX=${DEVSTACK_GATE_FEATURE_MATRIX:-roles/test-matrix/files/features.yaml}
 
 # Set to 1 to install, configure and enable the Tempest test suite; more flags may be
 # required to be set to customize the test run, e.g. DEVSTACK_GATE_TEMPEST_STRESS=1

devstack-vm-gate.sh

@@ -312,8 +312,9 @@ function setup_localrc {
             test_matrix_role='subnode'
         fi
 
-        MY_ENABLED_SERVICES=$(cd $BASE/new/devstack-gate && $PYTHON_PATH ./test-matrix.py -b $branch_for_matrix -f $DEVSTACK_GATE_FEATURE_MATRIX -r $test_matrix_role)
-        local original_enabled_services=$(cd $BASE/new/devstack-gate && $PYTHON_PATH ./test-matrix.py -b $branch_for_matrix -f $DEVSTACK_GATE_FEATURE_MATRIX -r primary)
+        TEST_MATRIX='roles/test-matrix/library/test_matrix.py -n'
+        MY_ENABLED_SERVICES=$(cd $BASE/new/devstack-gate && $PYTHON_PATH $TEST_MATRIX -b $branch_for_matrix -f $DEVSTACK_GATE_FEATURE_MATRIX -r $test_matrix_role)
+        local original_enabled_services=$(cd $BASE/new/devstack-gate && $PYTHON_PATH $TEST_MATRIX -b $branch_for_matrix -f $DEVSTACK_GATE_FEATURE_MATRIX -r primary)
         echo "MY_ENABLED_SERVICES: ${MY_ENABLED_SERVICES}"
         echo "original_enabled_services: ${original_enabled_services}"
 

roles/test-matrix/README.rst

@@ -0,0 +1,28 @@
+Set the a enabled_services fact based based on the test matrix
+
+**Role Variables**
+
+.. zuul:rolevar:: test_matrix_features
+   :default: files/features.yaml
+
+   The YAML file that defines the test matrix.
+
+.. zuul:rolevar:: test_matrix_branch
+   :default: {{ zuul.override_checkout | default(zuul.branch) }}
+
+   The git branch for which to calculate the test matrix.
+
+.. zuul:rolevar:: test_matrix_role
+   :default: primary
+
+   The role of the node for which the test matrix is calculated.
+   Valid values are 'primary' and 'subnode'.
+
+ .. zuul:rolevar:: test_matrix_configs
+    :default: []
+    :type: list
+
+   Feature configuration for the test matrix. This option allows enabling
+   more features, as defined in ``test_matrix_features``.
+   The default value is an empty list, however 'neutron' is added by default
+   from stable/ocata onwards.

roles/test-matrix/defaults/main.yaml

@@ -0,0 +1,4 @@
+test_matrix_features: files/features.yaml
+test_matrix_branch: "{{ zuul.override_checkout | default(zuul.branch) }}"
+test_matrix_role: primary
+test_matrix_configs: []

roles/test-matrix/library/test_matrix.py

@@ -121,7 +121,7 @@ of environmental feature definitions and flags.
 """
     parser = argparse.ArgumentParser(description=usage)
     parser.add_argument('-f', '--features',
-                        default='features.yaml',
+                        default='roles/test-matrix/files/features.yaml',
                         help="Yaml file describing the features matrix")
     parser.add_argument('-b', '--branch',
                         default="master",
@@ -133,6 +133,15 @@ of environmental feature definitions and flags.
                         default='primary',
                         help="What role this node will have",
                         choices=['primary', 'subnode'])
+    parser.add_argument('-a', '--ansible',
+                        dest='ansible',
+                        help="Behave as an Ansible Module",
+                        action='store_true')
+    parser.add_argument('-n', '--not-ansible',
+                        dest='ansible',
+                        help="Behave as python CLI",
+                        action='store_false')
+    parser.set_defaults(ansible=True)
     return parser.parse_args()
 
 
@@ -140,19 +149,47 @@ def main():
     global GRID
     global ALLOWED_BRANCHES
     opts = get_opts()
-    GRID = parse_features(opts.features)
-    ALLOWED_BRANCHES = GRID['branches']['allowed']
-    branch = normalize_branch(opts.branch)
-    role = opts.role
+    if opts.ansible:
+        ansible_module = get_ansible_module()
+        features = ansible_module.params['features']
+        branch = ansible_module.params['branch']
+        role = ansible_module.params['role']
+        configs = ansible_module.params['configs']
+    else:
+        features = opts.features
+        branch = opts.branch
+        role = opts.role
+        configs = configs_from_env()
 
-    features = calc_features(branch, configs_from_env())
+    GRID = parse_features(features)
+    ALLOWED_BRANCHES = GRID['branches']['allowed']
+    branch = normalize_branch(branch)
+
+    features = calc_features(branch, configs)
     LOG.debug("Features: %s " % features)
 
     services = calc_services(branch, features, role)
     LOG.debug("Services: %s " % services)
 
-    if opts.mode == "services":
-        print(",".join(services))
+    if opts.ansible:
+        ansible_module.exit_json(changed='True', services=services)
+    else:
+        if opts.mode == "services":
+            print(",".join(services))
+
+
+def get_ansible_module():
+
+    from ansible.module_utils.basic import AnsibleModule
+
+    return AnsibleModule(
+        argument_spec=dict(
+            features=dict(type='str'),
+            branch=dict(type='str'),
+            role=dict(type='str'),
+            configs=dict(type='list')
+        )
+    )
 
 
 if __name__ == "__main__":

roles/test-matrix/tasks/main.yaml

@@ -0,0 +1,37 @@
+- name: Deploy then features matrix and d-g bash functions
+  copy:
+    src: "{{ test_matrix_features }}"
+    dest: "{{ ansible_user_dir }}"
+
+- name: Ensure virtualenv is installed
+  become: true
+  package:
+    name: virtualenv
+    state: present
+  when: ansible_os_family != 'Darwin'
+
+- name: Install PyYAML to parse the test matrix
+  pip:
+    name: PyYAML
+    virtualenv: "/tmp/.test_matrix_venv"
+
+- name: Append neutron to configs for stable/ocata+
+  set_fact:
+    test_matrix_configs: "{{ test_matrix_configs }} + [ 'neutron' ]"
+  when:
+    - '"neutron" not in test_matrix_configs'
+    - test_matrix_branch | match("^(stable/[o-z].*|master)$")
+
+- name: Run the test matrix
+  test_matrix:
+    features: "{{ ansible_user_dir }}/{{ test_matrix_features | basename }}"
+    branch: "{{ test_matrix_branch }}"
+    role: "{{ test_matrix_role }}"
+    configs: "{{ test_matrix_configs }}"
+  vars:
+    ansible_python_interpreter: "/tmp/.test_matrix_venv/bin/python"
+  register: test_matrix_result
+
+- name: Set the enabled_services fact
+  set_fact:
+    enabled_services: "{{ test_matrix_result.services }}"

test-features.sh

@@ -40,27 +40,27 @@ function assert_list_equal {
 }
 
 function test_full_master {
-    local results=$(DEVSTACK_GATE_TEMPEST=1 ./test-matrix.py)
+    local results=$(DEVSTACK_GATE_TEMPEST=1 ./roles/test-matrix/library/test_matrix.py -n)
     assert_list_equal $TEMPEST_FULL_MASTER $results
 }
 
 function test_full_feature_ec {
-    local results=$(DEVSTACK_GATE_TEMPEST=1 ./test-matrix.py -b feature/ec)
+    local results=$(DEVSTACK_GATE_TEMPEST=1 ./roles/test-matrix/library/test_matrix.py -n -b feature/ec)
     assert_list_equal $TEMPEST_FULL_MASTER $results
 }
 
 function test_neutron_master {
-    local results=$(DEVSTACK_GATE_NEUTRON=1 DEVSTACK_GATE_TEMPEST=1 ./test-matrix.py)
+    local results=$(DEVSTACK_GATE_NEUTRON=1 DEVSTACK_GATE_TEMPEST=1 ./roles/test-matrix/library/test_matrix.py -n)
     assert_list_equal $TEMPEST_NEUTRON_MASTER $results
 }
 
 function test_heat_slow_master {
-    local results=$(DEVSTACK_GATE_TEMPEST_HEAT_SLOW=1 DEVSTACK_GATE_NEUTRON=1 DEVSTACK_GATE_TEMPEST=1 ./test-matrix.py)
+    local results=$(DEVSTACK_GATE_TEMPEST_HEAT_SLOW=1 DEVSTACK_GATE_NEUTRON=1 DEVSTACK_GATE_TEMPEST=1 ./roles/test-matrix/library/test_matrix.py -n)
     assert_list_equal $TEMPEST_HEAT_SLOW_MASTER $results
 }
 
 function test_grenade_new_master {
-    local results=$(DEVSTACK_GATE_TEMPEST_HEAT_SLOW=1 DEVSTACK_GATE_GRENADE=pullup DEVSTACK_GATE_TEMPEST=1 ./test-matrix.py)
+    local results=$(DEVSTACK_GATE_TEMPEST_HEAT_SLOW=1 DEVSTACK_GATE_GRENADE=pullup DEVSTACK_GATE_TEMPEST=1 ./roles/test-matrix/library/test_matrix.py -n)
     assert_list_equal $GRENADE_NEW_MASTER $results
 }