Prepare for cinder gate test

We want to be able to run cinderlib functional tests on Cinder gates, so
this patch adds a playbook and a tool to convert from Cinder's config
file to the YAML required by our functional tests.

The cinder-gate-run playbook uses cinderlib_ignore_errors variable to
define whether the playbook should fail on errors or should they just be
ignored.

The playbook stores the test run results on the common logs directory so
they are automatically saved by the Cinder gate jobs.

CONTRIBUTING.rst

@@ -125,7 +125,7 @@ Ready to contribute? Here's how to set up `cinderlib` for local development.
 
     $ mkdir temp
     $ cd temp
-    $ sudo ../tests/functional/lvm-prepare.sh
+    $ sudo ../tools/lvm-prepare.sh
 
    The default configuration for the functional tests can be found in the
    `tests/functional/lvm.yaml` file.  For additional information on this file
@@ -186,7 +186,7 @@ we can just call it from the location where we want to create the file:
 
 .. code-block:: shell
 
-    $ sudo tests/functional/lvm-prepare.sh
+    $ sudo tools/lvm-prepare.sh
 
 Now we can use this LVM backend in *cinderlib*:
 

playbooks/cinder-gate-run.yaml

@@ -0,0 +1,29 @@
+- hosts: all
+  become: True
+  vars:
+    devstack_base_dir: "{{ devstack_base_dir|default('/opt/stack') }}"
+    cinderlib_dir: "{{ cinderlib_dir }}|default({{ devstack_base_dir }}/cinderlib)"
+    cl_log_file: "{{ devstack_base_dir }}/logs/cinderlib.txt"
+    cinderlib_ignore_errors: "{{ cinderlib_ignore_errors }}|default(no)"
+  tasks:
+    - name: Create temporary config directory
+      tempfile:
+          state: directory
+          suffix: cinderlib
+      register: tempdir
+
+    - name: Convert Cinder's config to cinderlib functional test YAML
+      shell:
+        cmd: "{{ cinderlib_dir }}/tools/cinder-to-yaml.py /etc/cinder/cinder.conf {{ tempdir.path }}/cinderlib.yaml >{{ cl_log_file }} 2>&1"
+      ignore_errors: "{{ cinderlib_ignore_errors }}"
+      register: generate_config
+
+    - name: Run cinderlib functional tests
+      shell:
+        cmd: "unit2 discover -v -s tests/functional >>{{ cl_log_file }} 2>&1"
+        executable: /bin/bash
+        chdir: "{{ cinderlib_dir }}"
+      environment:
+        CL_FTEST_CFG: "{{ tempdir.path }}/cinderlib.yaml"
+      when: generate_config.rc != 0
+      ignore_errors: "{{ cinderlib_ignore_errors }}"

tests/functional/base_tests.py

@@ -13,7 +13,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
 import functools
 import os
 import subprocess
@@ -72,7 +71,8 @@ class BaseFunctTestCase(unittest2.TestCase):
             # NOTE(geguileo): For some drivers need to use a custom sudo script
             # to find virtualenv commands (ie: cinder-rtstool).
             path = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
-            cls.root_helper = os.path.join(path, 'virtualenv-sudo.sh')
+            sudo_tool = os.path.join(path, '../../tools/virtualenv-sudo.sh')
+            cls.root_helper = os.path.abspath(sudo_tool)
         else:
             cls.root_helper = 'sudo'
         cinderlib.setup(root_helper=cls.root_helper,

tools/cinder-to-yaml.py

@@ -0,0 +1,71 @@
+#!/bin/env python
+"""Generate functional tests YAML configuration files from Cinder config file
+
+Functional tests require a YAML file with the backend configuration parameters.
+To facilitate running them on a deployment that already has Cinder configured
+(ie: devstack) this program can translate from cinder.conf to a valid YAML
+file that can be used to run cinderlib functional tests.
+
+This program uses the oslo.config module to load configuration options instead
+of using configparser directly because drivers will need variables to have the
+right type (string, list, integer...), and the types are defined in the code
+using oslo.config.
+"""
+
+import sys
+import yaml
+
+from six.moves import configparser
+
+from cinder.cmd import volume
+volume.objects.register_all()  # noqa
+
+from cinder.volume import configuration as config
+from cinder.volume import manager
+
+
+def convert(cinder_source, yaml_dest):
+    result_cfgs = []
+
+    # Manually parse the Cinder configuration file so we know which options are
+    # set.
+    parser = configparser.ConfigParser()
+    parser.read(cinder_source)
+    enabled_backends = parser.get('DEFAULT', 'enabled_backends')
+    backends = [name.strip() for name in enabled_backends.split(',') if name]
+
+    volume.CONF(('--config-file', cinder_source), project='cinder')
+
+    for backend in backends:
+        options_present = parser.options(backend)
+
+        # Dynamically loading the driver triggers adding the specific
+        # configuration options to the backend_defaults section
+        cfg = config.Configuration(manager.volume_backend_opts,
+                                   config_group=backend)
+        driver_ns = cfg.volume_driver.rsplit('.', 1)[0]
+        __import__(driver_ns)
+
+        # Use the backend_defaults section to extract the configuration for
+        # options that are present in the backend section and add them to
+        # the backend section.
+        opts = volume.CONF._groups['backend_defaults']._opts
+        known_present_options = [opt for opt in options_present if opt in opts]
+        volume_opts = [opts[option]['opt'] for option in known_present_options]
+        cfg.append_config_values(volume_opts)
+
+        # Now retrieve the options that are set in the configuration file.
+        result_cfgs.append({option: cfg.safe_get(option)
+                            for option in known_present_options})
+
+    result = {'backends': result_cfgs}
+    # Write the YAML to the destination
+    with open(yaml_dest, 'w') as f:
+        yaml.dump(result, f)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        sys.stderr.write('Incorrect number of arguments\n')
+        exit(1)
+    convert(sys.argv[1], sys.argv[2])