tripleo-common/scripts/upgrade-non-controller.sh

#!/bin/bash
#set -x
# Copyright 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

#
# Utility script that will be invoked by the operator as part of the documented
# upgrades workflow. Those roles that have disable_upgrade_deployment set to
# true will have had a /root/tripleo_upgrade_node.sh script delivered during
# the composable ansible upgrade steps. This tripleo_upgrade_node.sh is then
# invoked over ssh by the operator using the simple utility delivered in this
# file. See -h for options.
#
set -eu
set -o pipefail
SCRIPT_NAME=$(basename $0)

#can make the upgrade script overridable (if a different target will be used)
UPGRADE_SCRIPT=${UPGRADE_SCRIPT:-/root/tripleo_upgrade_node.sh}
#allow override incase the ssh user is not 'heat-admin' - must be able to sudo
UPGRADE_NODE_USER=${UPGRADE_NODE_USER:-"heat-admin"}
UPGRADE_NODE=""
QUERY_NODE=""
SCRIPT=""

function show_options {
    echo "Usage: $SCRIPT_NAME"
    echo
    echo "Options:"
    echo "  -h|--help                    -- print this help."
    echo "  -u|--upgrade <nova node>     -- nova node name or id to upgrade"
    echo "  -s|--script <absolute_path>  -- absolute path to the script you wish"
    echo "                                  to use for the upgrade"
    echo "  -q|--query <nova node>       -- determine if the node is ACTIVE and"
    echo "                                  has the upgrade script. Also, tail"
    echo "                                  yum.log for any package update info"
    echo
    echo "Invoke the /root/tripleo_upgrade_node.sh script on roles that have"
    echo " disable_upgrade_deployment flag set true, as part of the tripleo"
    echo " upgrade workflow. The tripleo_upgrade_node.sh is delivered when you"
    echo " execute the composable ansible upgrade steps. This utility is used"
    echo " by the operator to invoke the tripleo_upgrade_node.sh on a given"
    echo " named node (by nova name or uuid). Logfiles are generated in the"
    echo " current working directory by node name/uuid"
    echo
    echo "  Example invocations:"
    echo
    echo "    upgrade-non-controller.sh --upgrade overcloud-compute-0 "
    echo
    echo "  You can run on multiple nodes in parallel: "

    echo "    for i in \$(seq 0 2); do "
    echo "      upgrade-non-controller.sh --upgrade overcloud-compute-\$i &"
    echo "    # Remove the '&' above to have these upgrade in sequence"
    echo "    # rather than in parallel."
    echo
    exit $1
}

TEMP=`getopt -o h,u:,q:,s: -l help,upgrade:,query:,script: -n $SCRIPT_NAME -- "$@"`

if [ $? != 0 ]; then
    echo "Terminating..." >&2
    exit 1
fi

# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"

while true ; do
    case "$1" in
        -h | --help ) show_options 0 >&2;;
        -u | --upgrade ) UPGRADE_NODE="$2" ; shift 2 ;;
        -s | --script ) SCRIPT="$2"; shift 2 ;;
        -q | --query ) QUERY_NODE="$2" ; shift 2 ;;
        -- ) shift ; break ;;
        * ) echo "Error: unsupported option $1." ; exit 1 ;;
    esac
done

LOGFILE="$SCRIPT_NAME"-$UPGRADE_NODE$QUERY_NODE
function log {
  echo "`date` $SCRIPT_NAME $1" 2>&1 | tee -a  $LOGFILE
}
log "Logging to $LOGFILE"

function find_nova_node_by_name_or_id {
  name_or_id=$1
  node_status=$(openstack server show $name_or_id -f value -c status)
  if ! [[ $node_status == "ACTIVE" ]]; then
    log "ERROR: node $name_or_id not found to be ACTIVE. Current status is $node_status"
    exit 1
  fi
  log "nova node $name_or_id found with status $node_status"
}

function confirm_script_on_node {
  name_or_id=$1
  node_ip=$(nova show $name_or_id | grep "ctlplane network" | awk '{print $5}')
  log "checking for upgrade script $UPGRADE_SCRIPT on node $name_or_id ($node_ip)"
  results=$(ssh $UPGRADE_NODE_USER@$node_ip "sudo ls -l $UPGRADE_SCRIPT")
  log "upgrade script $UPGRADE_SCRIPT found on node $name_or_id ($node_ip)"
}

function deliver_script {
  script=$1
  node=$2
  node_ip=$(nova show $node | grep "ctlplane network" | awk '{print $5}')
  file_name=$(echo ${script##*/})
  log "Sending upgrade script $script to $node_ip as $UPGRADE_NODE_USER"
  scp $script $UPGRADE_NODE_USER@$node_ip:/home/$UPGRADE_NODE_USER/$file_name
  log "Copying upgrade script to right location and setting permissions"
  ssh $UPGRADE_NODE_USER@$node_ip "sudo cp /home/$UPGRADE_NODE_USER/$file_name $UPGRADE_SCRIPT ; \
                           sudo chmod 755 $UPGRADE_SCRIPT ; "
}

if [ -n "$UPGRADE_NODE" ]; then
  find_nova_node_by_name_or_id $UPGRADE_NODE
  if  [ -n "$SCRIPT" ]; then
    deliver_script $SCRIPT $UPGRADE_NODE
  fi
  confirm_script_on_node $UPGRADE_NODE
  node_ip=$(nova show $UPGRADE_NODE | grep "ctlplane network" | awk '{print $5}')
  log "Executing $UPGRADE_SCRIPT on $node_ip"
  ssh $UPGRADE_NODE_USER@$node_ip sudo $UPGRADE_SCRIPT 2>&1 | tee -a $LOGFILE
fi

if [ -n "$QUERY_NODE" ]; then
  # check node exists, check for upgrade script
  find_nova_node_by_name_or_id $QUERY_NODE
  confirm_script_on_node $QUERY_NODE
  node_ip=$(nova show $QUERY_NODE | grep "ctlplane network" | awk '{print $5}')
  log "We can't remotely tell if the upgrade has run on $QUERY_NODE."
  log "We can check for package updates... trying to tail yum.log on $QUERY_NODE:"
  ssh $UPGRADE_NODE_USER@$node_ip "sudo tail /var/log/yum.log" 2>&1 | tee -a $LOGFILE
fi