From fe2102f90c46808f4c3e3593b5dd1585bc20db47 Mon Sep 17 00:00:00 2001 From: Dmitry Tantsur Date: Thu, 26 Oct 2017 17:36:15 +0200 Subject: [PATCH] [devstack] stop setting or relying on standard properties This change moves our devstack plugin away from using the old scheduling properties: memory_mb, local_gb and cpus. Nova will stop using them for scheduling in Queens. The cpu_arch property is left intact, as it's still useful. The Placement API is polled to determine when nodes are fully exposed to nova instead of polling the hypervisor stats. cURL + jq are used, as the Placement OSC plugin is not ready yet. Change-Id: I497a992bd86e8a359f333d6936621d1136d68061 --- devstack/files/debs/ironic | 1 + devstack/files/rpms/ironic | 1 + devstack/lib/ironic | 69 ++++++++++++++++++++++------------- devstack/upgrade/resources.sh | 9 +++-- 4 files changed, 52 insertions(+), 28 deletions(-) diff --git a/devstack/files/debs/ironic b/devstack/files/debs/ironic index adf37951d6..1e85f6d345 100644 --- a/devstack/files/debs/ironic +++ b/devstack/files/debs/ironic @@ -37,3 +37,4 @@ squashfs-tools libvirt-dev socat ipxe-qemu +jq diff --git a/devstack/files/rpms/ironic b/devstack/files/rpms/ironic index ac22dd4e97..4f5987d385 100644 --- a/devstack/files/rpms/ironic +++ b/devstack/files/rpms/ironic @@ -20,3 +20,4 @@ libvirt-devel socat edk2-ovmf # dist:f24,f25 ipxe-roms-qemu +jq diff --git a/devstack/lib/ironic b/devstack/lib/ironic index 7f7bf44cd7..cd9ef9ee7b 100644 --- a/devstack/lib/ironic +++ b/devstack/lib/ironic @@ -184,7 +184,6 @@ IRONIC_VM_LOG_DIR=${IRONIC_VM_LOG_DIR:-$IRONIC_DATA_DIR/logs/} IRONIC_VM_LOG_ROTATE=$(trueorfalse True IRONIC_VM_LOG_ROTATE) # Set resource_classes for nodes to use Nova's placement engine -IRONIC_USE_RESOURCE_CLASSES=$(trueorfalse True IRONIC_USE_RESOURCE_CLASSES) IRONIC_DEFAULT_RESOURCE_CLASS=${IRONIC_DEFAULT_RESOURCE_CLASS:-baremetal} # Whether to build the ramdisk or download a prebuilt one. @@ -1510,26 +1509,56 @@ function create_bridge_and_vms { } function wait_for_nova_resources { - # TODO(jroll) if IRONIC_USE_RESOURCE_CLASSES, use the placement engine instead - # After nodes have been enrolled, we need to wait for both ironic and # nova's periodic tasks to populate the resource tracker with available # nodes and resources. Wait up to 2 minutes for a given resource before # timing out. - local resource=$1 - local expected_count=$2 + local expected_count=$1 + local resource_class=${IRONIC_DEFAULT_RESOURCE_CLASS^^} + + # TODO(dtantsur): switch to Placement OSC plugin, once it exists + local token + token=$(openstack token issue -f value -c id) + local endpoint + endpoint=$(openstack endpoint list --service placement --interface public -f value -c URL) + die_if_not_set $LINENO endpoint "Cannot find Placement API endpoint" + local i - echo_summary "Waiting 2 minutes for Nova resource tracker to pick up $resource >= $expected_count" + local count + echo_summary "Waiting 2 minutes for Nova resource tracker to pick up $expected_count nodes" for i in $(seq 1 120); do - if [ $(openstack hypervisor stats show -f value -c $resource) -ge $expected_count ]; then + # Fetch provider UUIDs from Placement + local providers + providers=$(curl -sH "X-Auth-Token: $token" $endpoint/resource_providers \ + | jq -r '.resource_providers[].uuid') + + local p + # Total count of the resource class, has to be equal to nodes count + count=0 + for p in $providers; do + local amount + # A resource class inventory record looks something like + # {"max_unit": 1, "min_unit": 1, "step_size": 1, "reserved": 0, "total": 1, "allocation_ratio": 1} + # Subtrack reserved from total (defaulting both to 0) + amount=$(curl -sH "X-Auth-Token: $token" $endpoint/resource_providers/$p/inventories \ + | jq ".inventories.CUSTOM_$resource_class as \$cls + | (\$cls.total // 0) - (\$cls.reserved // 0)") + + if [ $amount -gt 0 ]; then + count=$(( count + $amount )) + fi + done + + if [ $count -ge $expected_count ]; then return 0 fi + if is_service_enabled n-api; then $TOP_DIR/tools/discover_hosts.sh fi sleep 1 done - die $LINENO "Timed out waiting for Nova hypervisor-stats $resource >= $expected_count" + die $LINENO "Timed out waiting for Nova to track $expected_count nodes" } function _clean_ncpu_failure { @@ -1773,11 +1802,6 @@ function enroll_nodes { $IRONIC_CMD node manage $node_id --wait $IRONIC_MANAGE_TIMEOUT || \ die $LINENO "Node did not reach manageable state in $IRONIC_MANAGE_TIMEOUT seconds" - # TODO(dtantsur): only do it if IRONIC_USE_RESOURCE_CLASSES is False - $IRONIC_CMD node set $node_id --property cpus=$ironic_node_cpu \ - --property memory_mb=$ironic_node_ram \ - --property local_gb=$ironic_node_disk - # NOTE(vsaienko) IPA didn't automatically recognize root devices less than 4Gb. # Setting root hint allows to install OS on such devices. # 0x1af4 is VirtIO vendor device ID. @@ -1821,7 +1845,6 @@ function enroll_nodes { fi total_nodes=$((total_nodes+1)) - total_cpus=$((total_cpus+$ironic_node_cpu)) done < $ironic_hwinfo_file # NOTE(dtantsur): doing it outside of the loop, because of cleaning @@ -1832,13 +1855,12 @@ function enroll_nodes { local adjusted_disk adjusted_disk=$(($ironic_node_disk - $ironic_ephemeral_disk)) openstack flavor create --ephemeral $ironic_ephemeral_disk --ram $ironic_node_ram --disk $adjusted_disk --vcpus $ironic_node_cpu baremetal - if [[ "$IRONIC_USE_RESOURCE_CLASSES" == "True" ]]; then - local resource_class=${IRONIC_DEFAULT_RESOURCE_CLASS^^} - openstack flavor set baremetal --property "resources:CUSTOM_$resource_class"="1" - openstack flavor set baremetal --property "resources:DISK_GB"="0" - openstack flavor set baremetal --property "resources:MEMORY_MB"="0" - openstack flavor set baremetal --property "resources:VCPU"="0" - fi + + local resource_class=${IRONIC_DEFAULT_RESOURCE_CLASS^^} + openstack flavor set baremetal --property "resources:CUSTOM_$resource_class"="1" + openstack flavor set baremetal --property "resources:DISK_GB"="0" + openstack flavor set baremetal --property "resources:MEMORY_MB"="0" + openstack flavor set baremetal --property "resources:VCPU"="0" openstack flavor set baremetal --property "cpu_arch"="$ironic_node_arch" @@ -1858,11 +1880,8 @@ function enroll_nodes { # NOTE(vsaienko) we enrolling IRONIC_VM_COUNT on each node. So on subnode # we expect to have 2 x total_cpus total_nodes=$(( total_nodes * 2 )) - total_cpus=$(( total_cpus * 2 )) fi - wait_for_nova_resources "count" $total_nodes - # TODO(dtantsur): only do it when IRONIC_USE_RESOURCE_CLASSES is False - wait_for_nova_resources "vcpus" $total_cpus + wait_for_nova_resources $total_nodes fi } diff --git a/devstack/upgrade/resources.sh b/devstack/upgrade/resources.sh index 2b63a4a1ab..7b20b015f7 100755 --- a/devstack/upgrade/resources.sh +++ b/devstack/upgrade/resources.sh @@ -31,6 +31,9 @@ NEUTRON_NET=ironic_grenade set -o xtrace +# TODO(dtantsur): remove in Rocky, needed for parsing Placement API responses +install_package jq + function wait_for_ironic_resources { local i local nodes_count @@ -49,10 +52,10 @@ function wait_for_ironic_resources { die $LINENO "Timed out waiting for Ironic nodes are available again." } -total_cpus=$(( $IRONIC_VM_SPECS_CPU * $IRONIC_VM_COUNT )) +total_nodes=$IRONIC_VM_COUNT if [[ "${HOST_TOPOLOGY}" == "multinode" ]]; then - total_cpus=$(( 2 * $total_cpus )) + total_nodes=$(( 2 * $total_nodes )) fi function early_create { @@ -149,7 +152,7 @@ function destroy { case $1 in "early_create") wait_for_ironic_resources - wait_for_nova_resources "vcpus" $total_cpus + wait_for_nova_resources $total_nodes early_create ;; "create")