[ansible] Add basic root device hints support

This patch adds support for Ironic root hints to ansible deploy driver.

The advanced root hints based on oslo.utils operators are not supported yet,
only exact matches.

Change-Id: I6fd468cfa9a7792fa0e55aa58c337396e13d76ee

doc/source/drivers/ansible.rst

@@ -82,8 +82,11 @@ and partition images, on both ``msdos`` and ``GPT`` labeled disks.
 Root device hints
 ~~~~~~~~~~~~~~~~~
 
-Root device hints are currently not supported (first device returned as
-``ansible_devices`` fact is used), but support for them is planned.
+Root device hints are currently supported in their basic form only
+(with exact matches, without oslo.utils operators).
+If no root device hint is provided for the node, first device returned as
+part of ``ansible_devices`` fact is used as root device to create partitions
+on or write the whole disk image to.
 
 Node cleaning
 ~~~~~~~~~~~~~

ironic_staging_drivers/ansible/deploy.py

@@ -302,6 +302,38 @@ def _parse_partitioning_info(node):
     return {'partition_info': i_info}
 
 
+def _parse_root_device_hints(node):
+    """Convert string with hints to dict. """
+    root_device = node.properties.get('root_device')
+    if not root_device:
+        return {}
+    try:
+        parsed_hints = irlib_utils.parse_root_device_hints(root_device)
+    except ValueError as e:
+        raise exception.InvalidParameterValue(
+            _('Failed to validate the root device hints for node %(node)s. '
+              'Error: %(error)s') % {'node': node.uuid, 'error': e})
+    root_device_hints = {}
+    advanced = {}
+    for hint, value in parsed_hints.items():
+        if isinstance(value, six.string_types):
+            if value.startswith('== '):
+                root_device_hints[hint] = int(value[3:])
+            elif value.startswith('s== '):
+                root_device_hints[hint] = urlparse.unquote(value[4:])
+            else:
+                advanced[hint] = value
+        else:
+            root_device_hints[hint] = value
+    if advanced:
+        raise exception.InvalidParameterValue(
+            _('Ansible-deploy does not support advanced root device hints '
+              'based on oslo.utils operators. '
+              'Present advanced hints for node %(node)s are %(hints)s.') % {
+                  'node': node.uuid, 'hints': advanced})
+    return root_device_hints
+
+
 def _prepare_variables(task):
     node = task.node
     i_info = node.instance_info
@@ -333,6 +365,11 @@ def _prepare_variables(task):
             cfgdrv_type = 'file'
         variables['configdrive'] = {'type': cfgdrv_type,
                                     'location': cfgdrv_location}
+
+    root_device_hints = _parse_root_device_hints(node)
+    if root_device_hints:
+        variables['root_device_hints'] = root_device_hints
+
     return variables
 
 
@@ -441,6 +478,8 @@ class AnsibleDeploy(agent_base.HeartbeatMixin, base.DeployInterface):
         error_msg = _('Node %s failed to validate deploy image info. Some '
                       'parameters were missing') % node.uuid
         deploy_utils.check_for_missing_params(params, error_msg)
+        # validate root device hints, proper exceptions are raised from there
+        _parse_root_device_hints(node)
 
     def _ansible_deploy(self, task, node_address):
         """Internal function for deployment to a node."""

ironic_staging_drivers/ansible/playbooks/library/facts_wwn.py

@@ -0,0 +1,72 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+try:
+    import pyudev
+    HAS_PYUDEV = True
+except ImportError:
+    HAS_PYUDEV = False
+
+from ironic_staging_drivers.common.i18n import _LW
+
+
+COLLECT_INFO = (('wwn', 'WWN'), ('serial', 'SERIAL_SHORT'),
+                ('wwn_with_extension', 'WWN_WITH_EXTENSION'),
+                ('wwn_vendor_extension', 'WWN_VENDOR_EXTENSION'))
+
+
+def get_devices_wwn(devices):
+
+    if not HAS_PYUDEV:
+        LOG.warning(_LW('Can not collect "wwn", "wwn_with_extension", '
+                        '"wwn_vendor_extension" and "serial" when using '
+                        'root device hints because there\'s no UDEV python '
+                        'binds installed'))
+        return
+
+    dev_dict = {}
+    context = pyudev.Context()
+    for device in devices:
+        name = '/dev/' + device
+        try:
+            udev = pyudev.Device.from_device_file(context, name)
+        except (ValueError, EnvironmentError, pyudev.DeviceNotFoundError) as e:
+            LOG.warning(_LW('Device %(dev)s is inaccessible, skipping... '
+                            'Error: %(error)s'), {'dev': name, 'error': e})
+            continue
+
+        dev_dict[device] = {}
+        for key, udev_key in COLLECT_INFO:
+            dev_dict[device][key] = udev.get('ID_%s' % udev_key)
+
+    return {"ansible_facts": {"devices_wwn": dev_dict}}
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            devices=dict(required=True, type='list'),
+        ),
+        supports_check_mode=True,
+    )
+
+    devices = module.params['devices']
+    data = get_devices_wwn(devices)
+    module.exit_json(**data)
+
+
+from ansible.module_utils.basic import *  # noqa
+if __name__ == '__main__':
+    main()

ironic_staging_drivers/ansible/playbooks/library/root_hints.py

@@ -0,0 +1,98 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+GIB = 1 << 30
+
+EXTRA_PARAMS = set(['wwn', 'serial', 'wwn_with_extension',
+                    'wwn_vendor_extension'])
+
+
+# NOTE: ansible calculates device size as float with 2-digits precision,
+# Ironic requires size in GiB, if we will use ansible size parameter
+# a bug is possible for devices > 1 TB
+def size_gib(device_info):
+    sectors = device_info.get('sectors')
+    sectorsize = device_info.get('sectorsize')
+    if sectors is None or sectorsize is None:
+        return '0'
+
+    return str((int(sectors) * int(sectorsize)) // GIB)
+
+
+def merge_devices_info(devices, devices_wwn):
+    merged_info = devices.copy()
+    for device in merged_info:
+        if device in devices_wwn:
+            merged_info[device].update(devices_wwn[device])
+
+        # replace size
+        merged_info[device]['size'] = size_gib(merged_info[device])
+
+    return merged_info
+
+
+def root_hint(hints, devices):
+    hint = None
+    name = hints.pop('name', None)
+    for device in devices:
+        for key in hints:
+            if hints[key] != devices[device].get(key):
+                break
+        else:
+            # If multiple hints are specified, a device must satisfy all
+            # the hints
+            dev_name = '/dev/' + device
+            if name is None or name == dev_name:
+                hint = dev_name
+                break
+
+    return hint
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            root_device_hints=dict(required=True, type='dict'),
+            ansible_devices=dict(required=True, type='dict'),
+            ansible_devices_wwn=dict(required=True, type='dict')
+        ),
+        supports_check_mode=True)
+
+    hints = module.params['root_device_hints']
+    devices = module.params['ansible_devices']
+    devices_wwn = module.params['ansible_devices_wwn']
+
+    if devices_wwn is None:
+        extra = set(hints) & EXTRA_PARAMS
+        if extra:
+            module.fail_json(msg='Extra hints (supported by additional ansible'
+                             ' module) are set but this information can not be'
+                             ' collected. Extra hints: %s' % ', '.join(extra))
+
+    devices_info = merge_devices_info(devices, devices_wwn or {})
+    hint = root_hint(hints, devices_info)
+
+    if hint is None:
+        module.fail_json(msg='Root device hints are set, but none of the '
+                         'devices satisfy them. Collected devices info: %s'
+                         % devices_info)
+
+    ret_data = {'ansible_facts': {'ironic_root_device': hint}}
+    module.exit_json(**ret_data)
+
+
+from ansible.module_utils.basic import *  # noqa
+if __name__ == '__main__':
+    main()

ironic_staging_drivers/ansible/playbooks/roles/discover/tasks/main.yaml

@@ -1,3 +1,6 @@
+- include: roothints.yaml
+  when: ironic.root_device_hints is defined
+
 - set_fact:
     ironic_root_device: /dev/{{ ansible_devices.keys()[0] }}
   when: ironic_root_device is undefined

ironic_staging_drivers/ansible/playbooks/roles/discover/tasks/roothints.yaml

@@ -0,0 +1,8 @@
+- name: get devices wwn facts
+  facts_wwn:
+    devices: "{{ ansible_devices.keys() }}"
+- name: calculate root hint
+  root_hints:
+    root_device_hints: "{{ ironic.root_device_hints }}"
+    ansible_devices: "{{ ansible_devices }}"
+    ansible_devices_wwn: "{{ devices_wwn | default(None) }}"

ironic_staging_drivers/tests/unit/ansible/test_deploy.py

@@ -296,6 +296,37 @@ class TestAnsibleMethods(db_base.DbTestCase):
                 "foo": "bar"},
             ansible_deploy._prepare_extra_vars(host_list, ansible_vars))
 
+    def test__parse_root_device_hints(self):
+        hints = {"wwn": "fake wwn", "size": "12345", "rotational": True}
+        expected = {"wwn": "fake wwn", "size": 12345, "rotational": True}
+        props = self.node.properties
+        props['root_device'] = hints
+        self.node.properties = props
+        self.node.save()
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            self.assertEqual(
+                expected, ansible_deploy._parse_root_device_hints(task.node))
+
+    def test__parse_root_device_hints_fail_advanced(self):
+        hints = {"wwn": "s!= fake wwn",
+                 "size": ">= 12345",
+                 "name": "<or> spam <or> ham",
+                 "rotational": True}
+        expected = {"wwn": "s!= fake%20wwn",
+                    "name": "<or> spam <or> ham",
+                    "size": ">= 12345"}
+        props = self.node.properties
+        props['root_device'] = hints
+        self.node.properties = props
+        self.node.save()
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            exc = self.assertRaises(
+                exception.InvalidParameterValue,
+                ansible_deploy._parse_root_device_hints, task.node)
+            for key, value in expected.items():
+                self.assertIn(six.text_type(key), six.text_type(exc))
+                self.assertIn(six.text_type(value), six.text_type(exc))
+
     @mock.patch.object(ansible_deploy, '_calculate_memory_req', autospec=True,
                        return_value=2000)
     def test__prepare_variables(self, mem_req_mock):
@@ -308,6 +339,23 @@ class TestAnsibleMethods(db_base.DbTestCase):
             self.assertEqual(expected,
                              ansible_deploy._prepare_variables(task))
 
+    @mock.patch.object(ansible_deploy, '_calculate_memory_req', autospec=True,
+                       return_value=2000)
+    def test__prepare_variables_root_device_hints(self, mem_req_mock):
+        props = self.node.properties
+        props['root_device'] = {"wwn": "fake-wwn"}
+        self.node.properties = props
+        self.node.save()
+        expected = {"image": {"url": "http://image",
+                              "source": "fake-image",
+                              "mem_req": 2000,
+                              "disk_format": "qcow2",
+                              "checksum": "md5:checksum"},
+                    "root_device_hints": {"wwn": "fake-wwn"}}
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            self.assertEqual(expected,
+                             ansible_deploy._prepare_variables(task))
+
     @mock.patch.object(ansible_deploy, '_calculate_memory_req', autospec=True,
                        return_value=2000)
     def test__prepare_variables_noglance(self, mem_req_mock):

releasenotes/notes/ansible-driver-root-hints-f5ba5796f238e0a9.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Support for basic root device hints added to ansible deploy driver.
+    Only exact matches are supported for now, advanced hints based on
+    oslo.utils operators are not supported yet.