diff --git a/doc/source/howto.rst b/doc/source/howto.rst index 93df0ff53..1c0fd29db 100644 --- a/doc/source/howto.rst +++ b/doc/source/howto.rst @@ -230,7 +230,7 @@ pre-requisite software packages, Ansible, and then execute the to provide a single step testing mechanism. ``playbooks/test-bifrost-create-vm.yaml`` creates one or more VMs for -testing and saves out a baremetal.csv file which is used by +testing and saves out a baremetal.json file which is used by ``playbooks/test-bifrost.yaml`` to execute the remaining roles. Two additional roles are invoked by this playbook which enables Ansible to connect to the new nodes by adding them to the inventory, and then @@ -270,8 +270,8 @@ run virsh commands. test-bifrost-create-vm.yaml`` command to create a test virtual machine. #. Set the environment variable of ``BIFROST_INVENTORY_SOURCE`` to the - path to the csv file, which by default has been written to - /tmp/baremetal.csv. + path to the JSON file, which by default has been written to + /tmp/baremetal.json. #. Run the enrollment step, as documented above, using the CSV file you created in the previous step. #. Run the deployment step, as documented above. diff --git a/playbooks/roles/bifrost-create-vm-nodes/README.md b/playbooks/roles/bifrost-create-vm-nodes/README.md index bb16b7720..cf9eed65c 100644 --- a/playbooks/roles/bifrost-create-vm-nodes/README.md +++ b/playbooks/roles/bifrost-create-vm-nodes/README.md @@ -28,7 +28,17 @@ as extra-vars instead. Role Variables -------------- -baremetal_csv_file: "/tmp/baremetal.csv" +baremetal_csv_file: Deprecated. CSV file format is deprecated, and + this variable will be removed in the Queens release. + Use 'baremetal_json_file' variable instead. + Default is undefined. If defined, its value will be + used for 'baremetal_json_file' variable (see below), + although file created will still be in JSON format. + The driver assigned to nodes will be 'agent_ssh' + +baremetal_json_file: Defaults to '/tmp/baremetal.json' but will be overridden + by 'baremetal_csv_file' if that is defined. + The driver assigned to nodes will be 'agent_ssh' test_vm_memory_size: Tunable setting to allow a user to define a specific amount of RAM in MB to allocate to guest/test VMs. diff --git a/playbooks/roles/bifrost-create-vm-nodes/defaults/main.yml b/playbooks/roles/bifrost-create-vm-nodes/defaults/main.yml index d9c345cbc..8da05a64b 100644 --- a/playbooks/roles/bifrost-create-vm-nodes/defaults/main.yml +++ b/playbooks/roles/bifrost-create-vm-nodes/defaults/main.yml @@ -1,6 +1,6 @@ --- # defaults file for bifrost-create-vm-nodes -baremetal_csv_file: "/tmp/baremetal.csv" +baremetal_json_file: '/tmp/baremetal.json' test_vm_memory_size: "3072" test_vm_num_nodes: 1 test_vm_domain_type: "qemu" diff --git a/playbooks/roles/bifrost-create-vm-nodes/tasks/create_vm.yml b/playbooks/roles/bifrost-create-vm-nodes/tasks/create_vm.yml index f2d947740..345e00299 100644 --- a/playbooks/roles/bifrost-create-vm-nodes/tasks/create_vm.yml +++ b/playbooks/roles/bifrost-create-vm-nodes/tasks/create_vm.yml @@ -98,22 +98,29 @@ set_fact: vm_mac: "{{ (testvm_xml.get_xml | regex_findall(\"\") | first).split('=') | last | regex_replace(\"['/>]\", '') }}" -- name: set the csv entry for vm +- name: set the json entry for vm set_fact: - vm_csv_items: - - "{{ vm_mac }}" - - "root" - - "undefined" - - "192.168.122.1" - - "{{ test_vm_cpu_count }}" - - "{{ test_vm_memory_size }}" - - "{{ test_vm_disk_gib }}" - - "flavor" - - "type" - - "a8cb6624-0d9f-c882-affc-046ebb96ec0{{ testvm_csv_data | length + 1 }}" - - "{{ vm_name }}" - - "192.168.122.{{ testvm_csv_data | length + 2 }}" + testvm_data: + name: "{{ vm_name }}" + uuid: "{{ vm_name | to_uuid }}" + driver: "agent_ssh" + driver_info: + power: + ssh_address: "192.168.122.1" + ssh_port: "22" + ssh_username: "ironic" + ssh_key_filename: "/home/ironic/.ssh/id_rsa" + ssh_virt_type: "virsh" + nics: + - mac: "{{ vm_mac }}" + ansible_ssh_host: "192.168.122.{{ testvm_json_data | length + 2 }}" + ipv4_address: "192.168.122.{{ testvm_json_data | length + 2 }}" + properties: + cpu_arch: "{{ test_vm_arch }}" + ram: "{{ test_vm_memory_size }}" + cpus: "{{ test_vm_cpu_count }}" + disk_size: "{{ test_vm_disk_gib }}" - name: add created vm info set_fact: - testvm_csv_data: "{{ testvm_csv_data + [vm_csv_items] }}" + testvm_json_data: "{{ testvm_json_data | combine({vm_name: testvm_data}) }}" diff --git a/playbooks/roles/bifrost-create-vm-nodes/tasks/main.yml b/playbooks/roles/bifrost-create-vm-nodes/tasks/main.yml index bcd487dc7..7a789ccb1 100644 --- a/playbooks/roles/bifrost-create-vm-nodes/tasks/main.yml +++ b/playbooks/roles/bifrost-create-vm-nodes/tasks/main.yml @@ -12,6 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. --- +- name: produce warning when csv file is defined + debug: + msg: > + "WARNING - Variable 'baremetal_csv_file' is deprecated. + For backward compatibility, its value will be used as path for + file to write data for created 'virtual' baremetal nodes, + but the file will be JSON formatted." + when: baremetal_csv_file is defined + +- name: override baremetal_json_file with csv file path + set_fact: + baremetal_json_file: "{{ baremetal_csv_file }}" + when: baremetal_csv_file is defined + # NOTE(cinerama) openSUSE Tumbleweed & Leap have different distribution # IDs which are not currently accounted for in Ansible, so adjust facts # so we can have shared defaults for the whole SuSE family. @@ -81,37 +95,27 @@ - name: create placeholder var for vm entries in CSV format set_fact: - testvm_csv_data: [] + testvm_json_data: {} - include: create_vm.yml with_items: "{{ test_vm_node_names }}" -- name: remove previous baremetal csv file +- name: remove previous baremetal data file file: state: absent - path: "{{ baremetal_csv_file }}" + path: "{{ baremetal_json_file }}" -- name: create empty baremetal csv file - file: - state: touch - path: "{{ baremetal_csv_file }}" - -# NOTE(pas-ha) this is a weird Ansible way to not flatten list of lists -- name: write to baremetal csv file - lineinfile: - state: present - name: "{{ baremetal_csv_file }}" - line: "{{ item | join(',') }}" - with_nested: - - "{{ testvm_csv_data }}" +- name: write to baremetal json file + copy: + dest: "{{ baremetal_json_file }}" + content: "{{ testvm_json_data | to_nice_json }}" - name: > - "Set file permissions such that the baremetal csv file at /tmp/baremetal.csv + "Set file permissions such that the baremetal data file can be read by the user executing Ansible" file: - path: "{{ baremetal_csv_file }}" + path: "{{ baremetal_json_file }}" owner: "{{ ansible_env.SUDO_USER }}" when: > ansible_env.SUDO_USER is defined and - baremetal_csv_file is defined and - baremetal_csv_file != "" + baremetal_json_file != "" diff --git a/playbooks/roles/bifrost-test-dhcp/files/test-dhcp.py b/playbooks/roles/bifrost-test-dhcp/files/test-dhcp.py index b14887b1f..939fda871 100644 --- a/playbooks/roles/bifrost-test-dhcp/files/test-dhcp.py +++ b/playbooks/roles/bifrost-test-dhcp/files/test-dhcp.py @@ -17,10 +17,34 @@ from __future__ import print_function import csv +import json import os import sys +def _load_data_from_csv(path): + with open(path) as csvfile: + csvdata = [row for row in csv.reader(csvfile)] + inventory = {} + # NOTE(pas-ha) convert to structure similar to JSON inventory + for entry in csvdata: + mac = entry[0] + hostname = entry[10] + ip = entry[11] + inventory[hostname] = { + 'nics': [{'mac': mac}], + 'name': hostname, + 'ipv4_address': ip + } + return inventory + + +def _load_data_from_json(path): + with open(path) as jsonfile: + inventory = json.load(jsonfile) + return inventory + + def main(argv): # first item is the inventory_dhcp setting # second item is the inventory_dhcp_static_ip setting @@ -31,17 +55,20 @@ def main(argv): # nothing to validate sys.exit(0) - # extract data from csv file - inventory = [] - if not os.path.exists('/tmp/baremetal.csv'): + # load data from json file + if os.path.exists('/tmp/baremetal.json'): + inventory = _load_data_from_json('/tmp/baremetal.json') + # load data from csv file + elif os.path.exists('/tmp/baremetal.csv'): + try: + inventory = _load_data_from_csv('/tmp/baremetal.csv') + except Exception: + # try load *.csv as json for backward compatibility + inventory = _load_data_from_json('/tmp/baremetal.csv') + else: print('ERROR: Inventory file has not been generated') sys.exit(1) - with open('/tmp/baremetal.csv') as csvfile: - inventory_reader = csv.reader(csvfile) - for row in inventory_reader: - inventory.append(row) - # now check that we only have these entries in leases file leases = [] if not os.path.exists('/var/lib/misc/dnsmasq.leases'): @@ -59,27 +86,23 @@ def main(argv): sys.exit(1) # then we check that all macs and hostnames are present - for entry in inventory: - mac = entry[0] - hostname = entry[10] - ip = entry[11] + for value in inventory.values(): + # NOTE(pas-ha) supporting only single nic + mac = value['nics'][0]['mac'] + hostname = value['name'] + ip = value['ipv4_address'] # mac check - found = False for lease_entry in leases: if lease_entry[1] == mac: - found = True break - if not found: + else: print('ERROR: No mac found in leases') sys.exit(1) # hostname check - found = False for lease_entry in leases: if lease_entry[3] == hostname: - found = True - # if we use static ip, we need to check that ip matches # with hostname in leases if inventory_dhcp_static_ip: @@ -87,7 +110,7 @@ def main(argv): print('ERROR: IP does not match with inventory') sys.exit(1) break - if not found: + else: print('ERROR: No hostname found in leases') sys.exit(1) diff --git a/playbooks/test-bifrost-create-vm.yaml b/playbooks/test-bifrost-create-vm.yaml index 4bf026fb9..c2f723910 100644 --- a/playbooks/test-bifrost-create-vm.yaml +++ b/playbooks/test-bifrost-create-vm.yaml @@ -6,10 +6,21 @@ become: yes gather_facts: yes pre_tasks: - - name: "Set default baremetal.csv file if not already defined" + - name: "Warn if baremetal_csv_file is defined" + debug: + msg: > + "WARNING - 'baremetal_csv_file' variable is defined. + Its use is deprecated. The file created will be in JSON format. + Use 'baremetal_json_file' variable instead." + when: baremetal_csv_file is defined + - name: "Re-set baremetal json to csv file if defined" set_fact: - baremetal_csv_file: "/tmp/baremetal.csv" - when: baremetal_csv_file is not defined + baremetal_json_file: "{{ baremetal_csv_file }}" + when: baremetal_csv_file is defined + - name: "Set default baremetal.json file if not already defined" + set_fact: + baremetal_json_file: "/tmp/baremetal.json" + when: baremetal_json_file is not defined - name: "Set ci_testing flag if a list of changes are found in the environment variables" set_fact: ci_testing: true diff --git a/playbooks/test-bifrost.yaml b/playbooks/test-bifrost.yaml index 87a5bf309..c7803a28d 100644 --- a/playbooks/test-bifrost.yaml +++ b/playbooks/test-bifrost.yaml @@ -2,7 +2,7 @@ # Create a VM: # ansible-playbook -vvvv -i inventory/localhost test-bifrost-create-vm.yaml # Set BIFROST_INVENTORY_SOURCE -# export BIFROST_INVENTORY_SOURCE=/tmp/baremetal.csv +# export BIFROST_INVENTORY_SOURCE=/tmp/baremetal.json # Execute the installation and VM startup test. # ansible-playbook -vvvv -i inventory/bifrost_inventory.py test-bifrost.yaml -e use_cirros=true -e testing_user=cirros --- @@ -143,16 +143,20 @@ # The following tasks are intended to test DHCP functionality - hosts: localhost connection: local - name: "Executes DHCP test script" + name: "Start VMs that were not enrolled to ironic" become: yes + vars: + not_enrolled_data_file: /tmp/baremetal.json.rest tasks: # NOTE(TheJulia): Moved the power ON of the excess VMs until after # the other test VMs have been shutdown, in order to explicitly # validate that the dhcp config is working as expected and not # serving these requests. - name: Power on remaining test VMs - command: virsh start testvm{{item}} - with_sequence: start=4 end={{ test_vm_num_nodes | default('5') }} + virt: + name: "{{item.key}}" + state: running + with_dict: "{{ lookup('file', not_enrolled_data_file) | from_json }}" ignore_errors: yes when: inventory_dhcp | bool == true - name: Wait 30 seconds diff --git a/releasenotes/notes/test-with-json-inventory-b05204009f880431.yaml b/releasenotes/notes/test-with-json-inventory-b05204009f880431.yaml new file mode 100644 index 000000000..4138cc808 --- /dev/null +++ b/releasenotes/notes/test-with-json-inventory-b05204009f880431.yaml @@ -0,0 +1,30 @@ +--- +features: + - | + Bifrost's testing has been moved to use JSON-formatted baremetal inventory + file instead of deprecated CSV-formatted one. + The ``bifrost-create-vm-nodes`` role still accepts ``baremetal_csv_file`` + variable as path to where to write inventory, but the file content will + always be in JSON format. + A new variable ``baremetal_json_file`` should instead be used + as a location to where to write the test baremetal inventory file. +upgrade: + - | + The ``baremetal_csv_file`` variable in ``bifrost-create-vm-nodes`` role + has been deprecated and will be removed in the Queens release. + The inventory file written to this location by this role is now always + in JSON format. + The variable ``baremetal_json_file`` should be used instead of + ``baremetal_csv_file``. + This concerns only those operators who run tests for bifrost on + virtual hardware using ``bifrost-create-vm-nodes`` role and out-of-tree + scripts to process the baremetal inventory file produced by this role. + If such scripts do rely on this file being in CSV format, + they must be updated to use JSON format instead. +deprecations: + - | + The CSV format for baremetal inventory file is deprecated and using + it will be impossible in the Queens release. + During deprecation period it's handling is still supported by bifrost's + dynamic inventory, but this functionality will be removed in the Queens + release. diff --git a/scripts/split_json.py b/scripts/split_json.py new file mode 100644 index 000000000..f923d422f --- /dev/null +++ b/scripts/split_json.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# +# Copyright (c) 2017 Mirantis Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper script to cut down number of etries in JSON baremetal data file. + +Splits the JSON file containing a top-level dict structure into two files, +where first has at most N entries, and the second has the rest of them. + +Uses OrderedDict to preserve ordering of dict elements in input JSON file. +""" + +from __future__ import print_function + +import collections +import json +import os +import sys + +HELP_MSG = """ +Usage: + python %s + +where: + N - max number of dict elements to be defined in the JSON file + input - path to input JSON file + - path to first output JSON file containig at most N entries + - path to the second output JSON file containig the rest +""" % sys.argv[0] + + +def fail(msg=None): + if msg: + print("Error: %s" % msg) + print() + print(HELP_MSG) + sys.exit(1) + + +def parse_args(args): + + if len(args) != 4: + fail("Wrong number of arguments") + num, infile, out1, out2 = args + + try: + num = int(num) + except ValueError: + fail("First argument is not integer") + + if not os.path.isfile(infile): + fail("Input file %s does not exist or can not be accessed." % infile) + return num, infile, out1, out2 + + +def write_to_json(fname, data): + with open(fname, 'w') as of: + try: + json.dump(data, of, indent=4) + except Exception as ex: + fail("Failed to save data to %s file - Error %s" % (fname, ex)) + + +def split_json_dict(args): + num, infile, out1, out2 = parse_args(args) + data = {} + with open(infile) as f: + data = json.load(f, object_pairs_hook=collections.OrderedDict) + if not data: + fail("Baremetal data file %s is empty or non-valid JSON." % infile) + + first_data = collections.OrderedDict() + second_data = collections.OrderedDict() + for i, k in enumerate(data): + if i < num: + first_data[k] = data[k] + else: + second_data[k] = data[k] + + write_to_json(out1, first_data) + write_to_json(out2, second_data) + + +if __name__ == '__main__': + split_json_dict(sys.argv[1:]) + sys.exit(0) diff --git a/scripts/test-bifrost.sh b/scripts/test-bifrost.sh index 91dc1b94c..dc82be2eb 100755 --- a/scripts/test-bifrost.sh +++ b/scripts/test-bifrost.sh @@ -10,6 +10,7 @@ ENABLE_VENV="false" USE_DHCP="false" USE_VENV="false" BUILD_IMAGE="false" +BAREMETAL_DATA_FILE=${BAREMETAL_DATA_FILE:-'/tmp/baremetal.json'} # Set defaults for ansible command-line options to drive the different # tests. @@ -157,17 +158,23 @@ ${ANSIBLE} -vvvv \ -e test_vm_num_nodes=${TEST_VM_NUM_NODES} \ -e test_vm_memory_size=${VM_MEMORY_SIZE} \ -e test_vm_domain_type=${VM_DOMAIN_TYPE} \ + -e baremetal_json_file=${BAREMETAL_DATA_FILE} \ -e enable_venv=${ENABLE_VENV} if [ ${USE_DHCP} = "true" ]; then - # cut file to limit number of nodes to enroll for testing purposes - head -n -2 /tmp/baremetal.csv > /tmp/baremetal.csv.new && mv /tmp/baremetal.csv.new /tmp/baremetal.csv + # reduce the number of nodes in JSON file + # to limit number of nodes to enroll for testing purposes + python $BIFROST_HOME/scripts/split_json.py 3 \ + ${BAREMETAL_DATA_FILE} \ + ${BAREMETAL_DATA_FILE}.new \ + ${BAREMETAL_DATA_FILE}.rest \ + && mv ${BAREMETAL_DATA_FILE}.new ${BAREMETAL_DATA_FILE} fi set +e # Set BIFROST_INVENTORY_SOURCE -export BIFROST_INVENTORY_SOURCE=/tmp/baremetal.csv +export BIFROST_INVENTORY_SOURCE=${BAREMETAL_DATA_FILE} # Execute the installation and VM startup test. @@ -189,6 +196,7 @@ ${ANSIBLE} -vvvv \ -e noauth_mode=${NOAUTH_MODE} \ -e enable_keystone=${ENABLE_KEYSTONE} \ -e wait_for_node_deploy=${WAIT_FOR_DEPLOY} \ + -e not_enrolled_data_file=${BAREMETAL_DATA_FILE}.rest \ ${CLOUD_CONFIG} EXITCODE=$?