inventory: Add support for defining custom host groups

Add new 'host_groups' property in inventory files to add hosts to
additional Ansible groups. Moreover, add new DEFAULT_HOST_GROUPS env
variable which can be used to override the default 'baremetal' group.
Finally, add two new variables 'test_vm_default_groups' and
'test_vm_groups' to allow users to set per-vm and default inventory
groups when provisioning virtual machines.

Change-Id: Id82e63e7049072437a5a8ad5c0300fb29f75ae1f

bifrost/inventory.py

@@ -22,7 +22,7 @@ Bifrost Inventory Module
 This is a dynamic inventory module intended to provide a platform for
 consistent inventory information for Bifrost.
 
-The inventory currently supplies two distinct groups:
+The inventory supplies two distinct groups by default:
 
     - localhost
     - baremetal
@@ -35,6 +35,21 @@ variables extracted from the data source. The variables are defined on a
 per-host level which allows explicit actions to be taken based upon the
 variables.
 
+It is also possible for users to specify additional per-host groups by
+simply setting the host_groups variable in the inventory file. See below for
+an example JSON file.
+
+The default group can also be changed by setting the DEFAULT_HOST_GROUPS
+variable to contain the desired groups separated by whitespace as follows:
+
+DEFAULT_HOST_GROUPS="foo bar zoo"
+
+In case of provisioning virtual machines, additional per-VM groups can
+be set by simply setting the test_vm_groups[$host] variable to a list
+of desired groups. Moreover, users can override the default 'baremetal'
+group by assigning a list of default groups to the test_vm_default_group
+variable.
+
 Presently, the base mode of operation reads a CSV file in the format
 originally utilized by bifrost and returns structured JSON that is
 interpreted by Ansible.  This has since been extended to support the
@@ -62,6 +77,10 @@ Example JSON Element:
 {
   "node1": {
     "uuid": "a8cb6624-0d9f-c882-affc-046ebb96ec01",
+    "host_groups": [
+        "nova",
+        "neutron"
+    ],
     "driver_info": {
       "power": {
         "ipmi_target_channel": "0",
@@ -181,6 +200,11 @@ def _process_baremetal_data(data_source, groups, hostvars):
         # Perform basic validation
         node_net_data = host.get('node_network_data')
         ipv4_addr = host.get('ipv4_address')
+        default_groups = os.environ.get('DEFAULT_HOST_GROUPS',
+                                        'baremetal').split()
+        host['host_groups'] = sorted(list(set(host.get('host_groups', []) +
+                                              default_groups)))
+
         if not node_net_data and not ipv4_addr:
             host['addressing_mode'] = "dhcp"
         else:
@@ -190,7 +214,10 @@ def _process_baremetal_data(data_source, groups, hostvars):
                 'addressing_mode' not in host):
             host['provisioning_ipv4_address'] = host['ipv4_address']
         # Add each host to the values to be returned.
-        groups['baremetal']['hosts'].append(host['name'])
+        for group in host['host_groups']:
+            if group not in groups:
+                groups.update({group: {'hosts': []}})
+            groups[group]['hosts'].append(host['name'])
         hostvars.update({host['name']: host})
     return (groups, hostvars)
 
@@ -226,6 +253,7 @@ def _process_baremetal_csv(data_source, groups, hostvars):
             properties['cpu_arch'] = "x86_64"
             host['uuid'] = _val_or_none(row, 9)
             host['name'] = _val_or_none(row, 10)
+            host['host_groups'] = ["baremetal"]
             host['ipv4_address'] = _val_or_none(row, 11)
             if ('ipv4_address' not in host or
                     not host['ipv4_address']):
@@ -402,6 +430,14 @@ def main():
         LOG.error('Failed processing: %s' % error)
         sys.exit(1)
 
+    # Drop empty groups. This is usually necessary when
+    # the default ["baremetal"] group has been overridden
+    # by the user.
+    for group in groups.keys():
+        # Empty groups
+        if len(groups[group]['hosts']) == 0:
+            del groups[group]
+
     # General Data Conversion
 
     if not config.convertcsv:

bifrost/tests/functional/test_inventory_functional.py

@@ -48,7 +48,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
  "ipmi_target_address": null, "ipmi_target_channel": null,
  "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
  [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}, "hostname0":
+ "x86_64", "disk_size": "1024", "cpus": "2"}, "host_groups": ["baremetal"]},
+ "hostname0":
  {"uuid": "00000000-0000-0000-0000-000000000001", "driver": "agent_ipmitool",
  "name": "hostname0", "ipv4_address": "192.168.1.2",
  "provisioning_ipv4_address": "192.168.1.2", "ansible_ssh_host":
@@ -57,7 +58,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
  "ipmi_target_address": null, "ipmi_target_channel": null,
  "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
  [{"mac": "00:01:02:03:04:05"}], "properties": {"ram": "8192",
- "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"}}}""".replace('\n', '')
+ "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"},
+ "host_groups": ["baremetal"]}}""".replace('\n', '')
         expected_groups = """{"baremetal": {"hosts": ["hostname0",
  "hostname1"]}, "localhost": {"hosts": ["127.0.0.1"]}}""".replace('\n', '')
 
@@ -80,7 +82,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
  "ipmi_transit_address": "40", "ipmi_transit_channel": "30",
  "ipmi_bridging": "dual"}}, "nics":
  [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}}""".replace('\n', '')
+ "x86_64", "disk_size": "1024", "cpus": "2"},
+ "host_groups": ["baremetal"]}}""".replace('\n', '')
 
         expected_groups = """{"baremetal": {"hosts": ["hostname1"]},
  "localhost": {"hosts": ["127.0.0.1"]}}""".replace('\n', '')
@@ -104,7 +107,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
  "ipmi_transit_address": null, "ipmi_transit_channel": null,
  "ipmi_bridging": "single"}}, "nics":
  [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}}""".replace('\n', '')
+ "x86_64", "disk_size": "1024", "cpus": "2"},
+ "host_groups": ["baremetal"]}}""".replace('\n', '')
 
         (groups, hostvars) = utils.bifrost_csv_conversion(CSV)
         self.assertDictEqual(json.loads(str(expected_hostvars)), hostvars)
@@ -123,7 +127,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1
  "ipmi_target_channel": null, "ipmi_transit_address": null,
  "ipmi_transit_channel": null}}, "nics":
  [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}}""".replace('\n', '')
+ "x86_64", "disk_size": "1024", "cpus": "2"},
+ "host_groups": ["baremetal"]}}""".replace('\n', '')
 
         (groups, hostvars) = utils.bifrost_csv_conversion(CSV)
         self.assertDictEqual(json.loads(str(expected_hostvars)), hostvars)
@@ -146,7 +151,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1
  "ipmi_target_channel": null, "ipmi_transit_address": null,
  "ipmi_transit_channel": null}}, "nics":
  [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}}""".replace('\n', '')
+ "x86_64", "disk_size": "1024", "cpus": "2"},
+ "host_groups": ["baremetal"]}}""".replace('\n', '')
 
         (groups, hostvars) = utils.bifrost_csv_conversion(CSV)
         self.assertDictEqual(json.loads(str(expected_hostvars)), hostvars)
@@ -170,7 +176,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
  "ipmi_target_address": null, "ipmi_target_channel": null,
  "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
  [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}, "hostname0":
+ "x86_64", "disk_size": "1024", "cpus": "2"}, "host_groups": ["baremetal"]},
+ "hostname0":
  {"uuid": "00000000-0000-0000-0000-000000000001", "driver": "agent_ipmitool",
  "name": "hostname0", "ipv4_address": "192.168.1.2", "ansible_ssh_host":
  "192.168.1.2", "provisioning_ipv4_address": "192.168.1.2",
@@ -179,7 +186,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
  "ipmi_target_address": null, "ipmi_target_channel": null,
  "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
  [{"mac": "00:01:02:03:04:05"}], "properties": {"ram": "8192",
- "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"}}}""".replace('\n', '')
+ "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"},
+ "host_groups": ["baremetal"]}}""".replace('\n', '')
 
         (groups, hostvars) = utils.bifrost_csv_conversion(CSV)
         self.assertDictEqual(json.loads(str(expected_hostvars)), hostvars)
@@ -200,7 +208,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
  "ipmi_target_address": null, "ipmi_target_channel": null,
  "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
  [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}, "hostname0":
+ "x86_64", "disk_size": "1024", "cpus": "2"}, "host_groups":
+ ["baremetal", "nova"]}, "hostname0":
  {"uuid": "00000000-0000-0000-0000-000000000001", "driver": "agent_ssh",
  "name": "hostname0", "ipv4_address": "192.168.1.2", "ansible_ssh_host":
  "192.168.1.2", "provisioning_ipv4_address": "192.168.1.2",
@@ -208,7 +217,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
  "ssh_key_filename": "/home/ironic/.ssh/id_rsa", "ssh_username":
  "ironic", "ssh_port": 22, "ssh_address": "192.0.2.2"}}, "nics":
  [{"mac": "00:01:02:03:04:05"}], "properties": {"ram": "8192",
- "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"}}}""".replace('\n', '')
+ "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"},
+ "host_groups": ["baremetal", "nova"]}}""".replace('\n', '')
         (groups, hostvars) = utils.bifrost_data_conversion(
             yaml.safe_dump(json.loads(str(expected_hostvars))))
         self.assertDictEqual(json.loads(str(expected_hostvars)), hostvars)
@@ -222,6 +232,6 @@ ipmi_password":"ADMIN"}},"driver":"agent_ipmitool"}}""".replace('\n', '')
 "00000000-0000-0000-0001-bad00000010","name":"h0000-01","driver_info"
 :{"power":{"ipmi_address":"10.0.0.78","ipmi_username":"ADMIN","
 ipmi_password":"ADMIN"}},"driver":"agent_ipmitool","addressing_mode":
-"dhcp"}}""".replace('\n', '')
+"dhcp","host_groups": ["baremetal"]}}""".replace('\n', '')
         (groups, hostvars) = utils.bifrost_data_conversion(input_json)
         self.assertDictEqual(json.loads(str(expected_json)), hostvars)

playbooks/roles/bifrost-create-vm-nodes/defaults/main.yml

@@ -5,6 +5,8 @@ test_vm_memory_size: "3072"
 test_vm_num_nodes: 1
 test_vm_domain_type: "qemu"
 test_vm_arch: "x86_64"
+test_vm_groups: {}
+test_vm_default_groups: "{{ lookup('env', 'DEFAULT_HOST_GROUPS').split() | default(['baremetal'], true) }}"
 test_vm_disk_gib: "{{ lookup('env', 'VM_DISK') | default(10, true) }}"
 test_vm_cpu_count: "{{ lookup('env', 'VM_CPU') | default(1, true) }}"
 test_vm_disk_cache: "{{ lookup('env', 'VM_DISK_CACHE') | default('writeback', true) }}"

playbooks/roles/bifrost-create-vm-nodes/tasks/create_vm.yml

@@ -18,6 +18,11 @@
 - set_fact:
     vm_name: "{{ item }}"
     vm_log_file: "{{ test_vm_logdir }}/{{ item }}_console.log"
+    vm_host_group: "{{ test_vm_default_groups }}"
+
+- set_fact:
+    vm_host_group: "{{ test_vm_default_groups | union(test_vm_groups[vm_name]) }}"
+  when: test_vm_groups[vm_name] is defined
 
 - name: set prealloc arg for Debian
   set_fact:
@@ -126,6 +131,7 @@
     testvm_data:
       name: "{{ vm_name }}"
       uuid: "{{ vm_name | to_uuid }}"
+      host_groups: "{{ vm_host_group }}"
       driver: "{{ test_vm_node_driver|default('agent_ipmitool') }}"
       driver_info:
         power:

releasenotes/notes/support-custom-ansible-groups-38ad5d3a930b17eb.yaml

@@ -0,0 +1,41 @@
+---
+features:
+  - |
+    It is now possible to define additional per-host inventory
+    groups for all the hosts that make use of the dynamic JSON
+    inventory. The way to do that is to simply define a
+    list of groups in the `host_group` property as illustrated
+    in the following example::
+
+    .. code-block:: yaml
+        "node1": {
+            "uuid": "a8cb6624-0d9f-c882-affc-046ebb96ec01",
+            "host_groups": [
+                "baremetal"
+            ],
+        }
+
+
+    When provisioning virtual machines it's possible to set
+    the per-VM inventory groups by setting the ``test_vm_host_groups``
+    variable as follows::
+
+    ``{ test_vm_host_groups: { testhost: [nova, cinder] } }``
+
+    It is also possible to change the default ``baremetal`` group
+    for virtual machines by simply setting the ``host_default_group``
+    variable to a list of default groups as follows::
+
+    ``{ test_vm_default_groups: [baremetal vms] }``
+
+    The list of default groups can also be set in the
+    ``DEFAULT_HOST_GROUPS`` environmental variable. This is currently
+    the only way to change the default group for baremetal hosts.
+
+    ``export DEFAULT_HOST_GROUPS="foo bar zoo"``
+
+    This will change the default groups to [foo, bar, zoo] instead of
+    the currently [baremetal] default. Extra care should be taken when
+    using this method since most bifrost playbooks depend on having a
+    [baremetal] group available for provisioning hosts.
+