refactor to remove functions-common

This refactor's the grenade code to remove the copy/pasted
functions-common from devstack and instead use the functions from
TARGET_DEVSTACK. In order to accomplish this we need to refactor when
the devstack trees are pulled, so that we always pull both the base
and target devstack trees *extremely* early.

We also need to include a small number of functions (mostly related to
git) in grenade to be able to bootstrap these trees. This is done in
the inc/bootstrap tree.

There are some slightly unrelated function moves into inc/upgrade to
consolidate all the upgrade functions which were scattered through the
codebase for clarity. inc/* will be the contract library interfaces
out of grenade (mirroring devstack).

To do this grenaderc has to be split between variables which can be
set without devstack functions, and variables we compute later. These
later variables will be moved into project specific directories based
on the plugin plan.

Change-Id: I1a2d1a1fed7858c4f3b51a19be2d12cc2d260a24
This commit is contained in:
Sean Dague 2015-04-06 10:50:56 -04:00
parent c32f052c2a
commit 6bb23b443f
8 changed files with 395 additions and 2183 deletions

View File

@ -7,8 +7,8 @@
# Include the common functions
FUNC_DIR=$(cd $(dirname "${BASH_SOURCE:-$0}") && pwd)
source ${FUNC_DIR}/functions-common
GRENADE_DIR=$(cd $(dirname "${BASH_SOURCE:-$0}") && pwd)
source ${GRENADE_DIR}/inc/upgrade
# Save trace setting
XTRACE=$(set +o | grep xtrace)
@ -42,43 +42,6 @@ function save_data {
done
}
# This function triggers the upgrade process for each project if it exists,
# otherwise it shows up a warning message about the lack of this file.
function upgrade_project {
# NOTE(maurosr): Ideally in a new upgrade test right after a release no new
# configuration is need, so we can go on without the from-<release> directory.
# This is also useful due to cross dependencie between d-g and grenade when
# enabling grenade to run a an upgrade between a new pair of releases.
project=$1
base_dir=$2
base_branch=$3
target_branch=$4
if [[ "$base_branch" == "$target_branch" ]]; then
direction="within"
else
direction="from"
fi
upgrade_dir=$(get_release_name_from_branch $base_branch)
upgrade_file=${base_dir}/${direction}"-"${upgrade_dir}/"upgrade-"${project}
if [[ -e ${upgrade_file} ]]; then
source ${upgrade_file} && configure_${project}_upgrade
else
echo "Warning: No new configurations were found for OpenStack $project."
echo "If your patch fails during the upgrade this may be the cause."
fi
}
# Determine whether grenade should be upgrading specified service, according
# to DO_NOT_UPGRADE_SERVICES
function should_upgrade {
if [[ "$DO_NOT_UPGRADE_SERVICES" =~ "$1" ]]; then
return 1
fi
return 0
}
# Restore xtrace
$XTRACE

File diff suppressed because it is too large Load Diff

View File

@ -21,13 +21,9 @@ GRENADE_DIR=$(cd $(dirname "$0") && pwd)
# Import common functions
source $GRENADE_DIR/functions
# Determine what system we are running on. This provides ``os_VENDOR``,
# ``os_RELEASE``, ``os_UPDATE``, ``os_PACKAGE``, ``os_CODENAME``
# and ``DISTRO``
GetDistro
# Source params
source $GRENADE_DIR/grenaderc
source $GRENADE_DIR/inc/bootstrap
RUN_BASE=$(trueorfalse True $RUN_BASE)
RUN_TARGET=$(trueorfalse True $RUN_TARGET)
@ -68,29 +64,6 @@ function stop {
fi
}
function upgrade_service {
local local_service=$1
# figure out if the service should be upgraded
echo "Checking for $local_service is enabled"
local enabled=""
# TODO(sdague) terrible work around because of missing
# devstack functions
if [[ $local_service == 'keystone' ]]; then
enabled="True"
else
enabled=$(
source $TARGET_DEVSTACK_DIR/functions;
source $TARGET_DEVSTACK_DIR/stackrc;
is_service_enabled $local_service || echo "False")
fi
if [[ "$enabled" == "False" ]]; then
echo_summary "Not upgrading $local_service"
return
fi
echo_summary "Upgrading $local_service..."
$GRENADE_DIR/upgrade-$local_service || die $LINENO "Failure in upgrade-$local_service"
}
# Ensure that we can run this on a fresh system
sudo mkdir -p $(dirname $BASE_DEVSTACK_DIR)
sudo mkdir -p $(dirname $TARGET_DEVSTACK_DIR)
@ -203,8 +176,23 @@ set -o errexit
set -o xtrace
# More Setup
# ==========
# Devstack Phase 2 initialization
# ===============================
#
# We now have enough infrastructure in grenade.sh to go and get *both*
# SOURCE and TARGET devstack trees. After which point we 'pivot' onto
# the TARGET devstack functions file, then source the rest of the
# grenade settings. This should let us run the bulk of grenade.
# get both devstack trees
fetch_devstacks
# get functions from Target Devstack
source $TARGET_DEVSTACK_DIR/functions
# source Phase 2 settings (which are dynamic). Realistically these are
# all going to migrate into project level settings.
source $GRENADE_DIR/grenaderc.settings
# Set up for smoke tests (default to True)
RUN_SMOKE=${RUN_SMOKE:=True}
@ -251,9 +239,6 @@ function run_javelin() {
# =================================
if [[ "$RUN_BASE" == "True" ]]; then
echo_summary "Running prep-base"
$GRENADE_DIR/prep-base
stop $STOP prep-base 01
echo_summary "Running base stack.sh"
cd $BASE_DEVSTACK_DIR

View File

@ -3,8 +3,6 @@
# Find the other rc files
RC_DIR=$(cd $(dirname "$BASH_SOURCE") && pwd)
source $RC_DIR/functions
# Base GIT Repo URL
# Another option is http://review.openstack.org/p
# Another option is https://github.com
@ -63,7 +61,6 @@ if [ -f $RC_DIR/localrc ]; then
source $RC_DIR/localrc
fi
# Defaults that use values from localrc go here
# Enable tempest
@ -80,30 +77,6 @@ TARGET_RUN_SMOKE=${TARGET_RUN_SMOKE:-$RUN_SMOKE}
# Projects that we should upgrade
UPGRADE_PROJECTS="keystone ceilometer swift glance neutron nova cinder horizon ironic"
# TODO(sdague): we really need to pull these in from devstack so we aren't
# always listing them.
BASE_SERVICES="nova-api nova-conductor keystone glance-api cinder-api"
BASE_SERVICES+=" swift-object-server swift-proxy-server"
# Partial nova upgrades (upgrade all but nova-compute)
if should_upgrade "n-cpu"; then
# only check for compute in base if we aren't supposed to shut it down
BASE_SERVICES+=" nova-compute"
else
TARGET_SERVICES="nova-compute "
fi
# Partial ironic upgrades (upgrade all but ironic)
if is_service_enabled "ironic"; then
if should_upgrade "ir-"; then
BASE_SERVICES+=" ironic-conductor ironic-api"
else
TARGET_SERVICES=" ironic-conductor ironic-api"
fi
fi
TARGET_SERVICES+=$BASE_SERVICES
# Need this for global requirements
REQUIREMENTS_DIR=$TARGET_RELEASE_DIR/requirements

42
grenaderc.settings Normal file
View File

@ -0,0 +1,42 @@
#!/bin/bash
#
# Copyright 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# These are late grenaderc settings that let us remove sourcing
# functions from grenaderc
# TODO(sdague): we really need to pull these in from devstack so we aren't
# always listing them.
BASE_SERVICES="nova-api nova-conductor keystone glance-api cinder-api"
BASE_SERVICES+=" swift-object-server swift-proxy-server"
# Partial nova upgrades (upgrade all but nova-compute)
if should_upgrade "n-cpu"; then
# only check for compute in base if we aren't supposed to shut it down
BASE_SERVICES+=" nova-compute"
else
TARGET_SERVICES="nova-compute "
fi
# Partial ironic upgrades (upgrade all but ironic)
if is_service_enabled "ironic"; then
if should_upgrade "ir-"; then
BASE_SERVICES+=" ironic-conductor ironic-api"
else
TARGET_SERVICES=" ironic-conductor ironic-api"
fi
fi
TARGET_SERVICES+=$BASE_SERVICES

255
inc/bootstrap Normal file
View File

@ -0,0 +1,255 @@
#!/bin/bash
#
# Copyright 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Bootstrapping Library for Grenade
# =================================
#
# We'd like grenade to largely use DEVSTACK_TARGET functions, but in
# order to do that we need to get to the point where we have a working
# DEVSTACK_TARGET. This file includes *just* enough functions to get
# us a working DEVSTACK_TARGET which then lets us pivot our
# functionality into there. Most of these will have been manually
# copied from upstream devstack.
# Normalize config values to True or False
# Accepts as False: 0 no No NO false False FALSE
# Accepts as True: 1 yes Yes YES true True TRUE
# VAR=$(trueorfalse default-value test-value)
function trueorfalse {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local default=$1
local literal=$2
local testval=${!literal:-}
[[ -z "$testval" ]] && { echo "$default"; return; }
[[ "0 no No NO false False FALSE" =~ "$testval" ]] && { echo "False"; return; }
[[ "1 yes Yes YES true True TRUE" =~ "$testval" ]] && { echo "True"; return; }
echo "$default"
$xtrace
}
# git clone only if directory doesn't exist already. Since ``DEST`` might not
# be owned by the installation user, we create the directory and change the
# ownership to the proper user.
# Set global ``RECLONE=yes`` to simulate a clone when dest-dir exists
# Set global ``ERROR_ON_CLONE=True`` to abort execution with an error if the git repo
# does not exist (default is False, meaning the repo will be cloned).
# Uses globals ``ERROR_ON_CLONE``, ``OFFLINE``, ``RECLONE``
# git_clone remote dest-dir branch
function git_clone {
local git_remote=$1
local git_dest=$2
local git_ref=$3
local orig_dir=$(pwd)
local git_clone_flags=""
RECLONE=$(trueorfalse False RECLONE)
if [[ "${GIT_DEPTH}" -gt 0 ]]; then
git_clone_flags="$git_clone_flags --depth $GIT_DEPTH"
fi
if [[ "$OFFLINE" = "True" ]]; then
echo "Running in offline mode, clones already exist"
# print out the results so we know what change was used in the logs
cd $git_dest
git show --oneline | head -1
cd $orig_dir
return
fi
if echo $git_ref | egrep -q "^refs"; then
# If our branch name is a gerrit style refs/changes/...
if [[ ! -d $git_dest ]]; then
[[ "$ERROR_ON_CLONE" = "True" ]] && \
die $LINENO "Cloning not allowed in this configuration"
git_timed clone $git_clone_flags $git_remote $git_dest
fi
cd $git_dest
git_timed fetch $git_remote $git_ref && git checkout FETCH_HEAD
else
# do a full clone only if the directory doesn't exist
if [[ ! -d $git_dest ]]; then
[[ "$ERROR_ON_CLONE" = "True" ]] && \
die $LINENO "Cloning not allowed in this configuration"
git_timed clone $git_clone_flags $git_remote $git_dest
cd $git_dest
# This checkout syntax works for both branches and tags
git checkout $git_ref
elif [[ "$RECLONE" = "True" ]]; then
# if it does exist then simulate what clone does if asked to RECLONE
cd $git_dest
# set the url to pull from and fetch
git remote set-url origin $git_remote
git_timed fetch origin
# remove the existing ignored files (like pyc) as they cause breakage
# (due to the py files having older timestamps than our pyc, so python
# thinks the pyc files are correct using them)
find $git_dest -name '*.pyc' -delete
# handle git_ref accordingly to type (tag, branch)
if [[ -n "`git show-ref refs/tags/$git_ref`" ]]; then
git_update_tag $git_ref
elif [[ -n "`git show-ref refs/heads/$git_ref`" ]]; then
git_update_branch $git_ref
elif [[ -n "`git show-ref refs/remotes/origin/$git_ref`" ]]; then
git_update_remote_branch $git_ref
else
die $LINENO "$git_ref is neither branch nor tag"
fi
fi
fi
# print out the results so we know what change was used in the logs
cd $git_dest
git show --oneline | head -1
cd $orig_dir
}
# git can sometimes get itself infinitely stuck with transient network
# errors or other issues with the remote end. This wraps git in a
# timeout/retry loop and is intended to watch over non-local git
# processes that might hang. GIT_TIMEOUT, if set, is passed directly
# to timeout(1); otherwise the default value of 0 maintains the status
# quo of waiting forever.
# usage: git_timed <git-command>
function git_timed {
local count=0
local timeout=0
if [[ -n "${GIT_TIMEOUT}" ]]; then
timeout=${GIT_TIMEOUT}
fi
until timeout -s SIGINT ${timeout} git "$@"; do
# 124 is timeout(1)'s special return code when it reached the
# timeout; otherwise assume fatal failure
if [[ $? -ne 124 ]]; then
die $LINENO "git call failed: [git $@]"
fi
count=$(($count + 1))
warn "timeout ${count} for git call: [git $@]"
if [ $count -eq 3 ]; then
die $LINENO "Maximum of 3 git retries reached"
fi
sleep 5
done
}
# git update using reference as a branch.
# git_update_branch ref
function git_update_branch {
local git_branch=$1
git checkout -f origin/$git_branch
# a local branch might not exist
git branch -D $git_branch || true
git checkout -b $git_branch
}
# git update using reference as a branch.
# git_update_remote_branch ref
function git_update_remote_branch {
local git_branch=$1
git checkout -b $git_branch -t origin/$git_branch
}
# git update using reference as a tag. Be careful editing source at that repo
# as working copy will be in a detached mode
# git_update_tag ref
function git_update_tag {
local git_tag=$1
git tag -d $git_tag
# fetching given tag only
git_timed fetch origin tag $git_tag
git checkout -f $git_tag
}
function fetch_devstacks {
# Fetch Base Devstack
# Get DevStack if it doesn't exist
if [[ ! -d $BASE_DEVSTACK_DIR ]]; then
git_clone $BASE_DEVSTACK_REPO $BASE_DEVSTACK_DIR $BASE_DEVSTACK_BRANCH
fi
# Load up a copy of the downloaded images if not present
if [[ -d ${STACK_ROOT}/images ]]; then
rsync -a ${STACK_ROOT}/images/* $BASE_DEVSTACK_DIR/files
fi
# Set up base localrc
# if localrc exists and localrc.orig does not exist, save localrc to localrc.orig
if [[ -r $BASE_DEVSTACK_DIR/localrc && ! -r $BASE_DEVSTACK_DIR/localrc.orig ]]; then
mv $BASE_DEVSTACK_DIR/localrc $BASE_DEVSTACK_DIR/localrc.orig
fi
# put devstack.localrc.target in place as localrc
sed -e "
s|\@BASE_RELEASE_DIR\@|$BASE_RELEASE_DIR|
s|\@DATA_DIR@|$DATA_DIR|
" $GRENADE_DIR/devstack.localrc.base >$BASE_DEVSTACK_DIR/localrc
# if localrc.orig exists, append it to localrc
if [[ -r $BASE_DEVSTACK_DIR/localrc.orig ]]; then
echo "#vvvvvvvvvv devstack-vm-gate.sh localrc vvvvvvvvvv" >>$BASE_DEVSTACK_DIR/localrc
cat $BASE_DEVSTACK_DIR/localrc.orig >>$BASE_DEVSTACK_DIR/localrc
fi
# if devstack.localrc exists append it to locarc
if [[ -r $GRENADE_DIR/devstack.localrc ]]; then
echo "#vvvvvvvvvv devstack.localrc vvvvvvvvvv" >>$BASE_DEVSTACK_DIR/localrc
cat $GRENADE_DIR/devstack.localrc >>$BASE_DEVSTACK_DIR/localrc
fi
# Fetch Target Devstack
# Get DevStack if it doesn't exist
if [[ ! -d $TARGET_DEVSTACK_DIR ]]; then
git_clone $TARGET_DEVSTACK_REPO $TARGET_DEVSTACK_DIR $TARGET_DEVSTACK_BRANCH
fi
# Set up target localrc
# if localrc exists and localrc.orig does not exist, save localrc to localrc.orig
if [[ -r $TARGET_DEVSTACK_DIR/localrc && ! -r $TARGET_DEVSTACK_DIR/localrc.orig ]]; then
mv $TARGET_DEVSTACK_DIR/localrc $TARGET_DEVSTACK_DIR/localrc.orig
fi
# put devstack.localrc.target in place as localrc
sed -e "
s|\@TARGET_RELEASE_DIR\@|$TARGET_RELEASE_DIR|
s|\@DATA_DIR@|$DATA_DIR|
" $GRENADE_DIR/devstack.localrc.target >$TARGET_DEVSTACK_DIR/localrc
# if localrc.orig exists, append it to localrc
if [[ -r $TARGET_DEVSTACK_DIR/localrc.orig ]]; then
echo "#vvvvvvvvvv devstack-vm-gate.sh localrc vvvvvvvvvv" >>$TARGET_DEVSTACK_DIR/localrc
cat $TARGET_DEVSTACK_DIR/localrc.orig >>$TARGET_DEVSTACK_DIR/localrc
fi
# if devstack.localrc exists append it to locarc
if [[ -r $GRENADE_DIR/devstack.localrc ]]; then
echo "#vvvvvvvvvv devstack.localrc vvvvvvvvvv" >>$TARGET_DEVSTACK_DIR/localrc
cat $GRENADE_DIR/devstack.localrc >>$TARGET_DEVSTACK_DIR/localrc
fi
}

78
inc/upgrade Normal file
View File

@ -0,0 +1,78 @@
#!/bin/bash
#
# Copyright 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
function upgrade_service {
local local_service=$1
# figure out if the service should be upgraded
echo "Checking for $local_service is enabled"
local enabled=""
# TODO(sdague) terrible work around because of missing
# devstack functions
if [[ $local_service == 'keystone' ]]; then
enabled="True"
else
enabled=$(
source $TARGET_DEVSTACK_DIR/functions;
source $TARGET_DEVSTACK_DIR/stackrc;
is_service_enabled $local_service || echo "False")
fi
if [[ "$enabled" == "False" ]]; then
echo_summary "Not upgrading $local_service"
return
fi
echo_summary "Upgrading $local_service..."
$GRENADE_DIR/upgrade-$local_service || die $LINENO "Failure in upgrade-$local_service"
}
# This function triggers the upgrade process for each project if it exists,
# otherwise it shows up a warning message about the lack of this file.
function upgrade_project {
# NOTE(maurosr): Ideally in a new upgrade test right after a release no new
# configuration is need, so we can go on without the from-<release> directory.
# This is also useful due to cross dependencie between d-g and grenade when
# enabling grenade to run a an upgrade between a new pair of releases.
project=$1
base_dir=$2
base_branch=$3
target_branch=$4
if [[ "$base_branch" == "$target_branch" ]]; then
direction="within"
else
direction="from"
fi
upgrade_dir=$(get_release_name_from_branch $base_branch)
upgrade_file=${base_dir}/${direction}"-"${upgrade_dir}/"upgrade-"${project}
if [[ -e ${upgrade_file} ]]; then
source ${upgrade_file} && configure_${project}_upgrade
else
echo "Warning: No new configurations were found for OpenStack $project."
echo "If your patch fails during the upgrade this may be the cause."
fi
}
# Determine whether grenade should be upgrading specified service, according
# to DO_NOT_UPGRADE_SERVICES
function should_upgrade {
if [[ "$DO_NOT_UPGRADE_SERVICES" =~ "$1" ]]; then
return 1
fi
return 0
}

View File

@ -25,52 +25,17 @@ set -o xtrace
# System Preparation
# ==================
# perform cleanup to ensure a clean starting environment
##add a config opt to still do this?
#rm -rf $TARGET_DEVSTACK_DIR
# Get DevStack if it doesn't exist
if [[ ! -d $TARGET_DEVSTACK_DIR ]]; then
git_clone $TARGET_DEVSTACK_REPO $TARGET_DEVSTACK_DIR $TARGET_DEVSTACK_BRANCH
fi
# Load up a copy of the downloaded images if not present
if [[ -d $BASE_RELEASE_DIR/images ]]; then
rsync -a $BASE_RELEASE_DIR/images $TARGET_DEVSTACK_DIR/files
fi
# Set up target localrc
# if localrc exists and localrc.orig does not exist, save localrc to localrc.orig
if [[ -r $TARGET_DEVSTACK_DIR/localrc && ! -r $TARGET_DEVSTACK_DIR/localrc.orig ]]; then
mv $TARGET_DEVSTACK_DIR/localrc $TARGET_DEVSTACK_DIR/localrc.orig
fi
# put devstack.localrc.target in place as localrc
sed -e "
s|\@TARGET_RELEASE_DIR\@|$TARGET_RELEASE_DIR|
s|\@DATA_DIR@|$DATA_DIR|
" $GRENADE_DIR/devstack.localrc.target >$TARGET_DEVSTACK_DIR/localrc
# if localrc.orig exists, append it to localrc
if [[ -r $TARGET_DEVSTACK_DIR/localrc.orig ]]; then
echo "#vvvvvvvvvv devstack-vm-gate.sh localrc vvvvvvvvvv" >>$TARGET_DEVSTACK_DIR/localrc
cat $TARGET_DEVSTACK_DIR/localrc.orig >>$TARGET_DEVSTACK_DIR/localrc
fi
# if devstack.localrc exists append it to locarc
if [[ -r $GRENADE_DIR/devstack.localrc ]]; then
echo "#vvvvvvvvvv devstack.localrc vvvvvvvvvv" >>$TARGET_DEVSTACK_DIR/localrc
cat $GRENADE_DIR/devstack.localrc >>$TARGET_DEVSTACK_DIR/localrc
fi
# Build a wheel cache
source $TARGET_DEVSTACK_DIR/stackrc
if [[ -n ${WHEELHOUSE:-} && ! -d ${WHEELHOUSE:-} ]]; then
WHEELHOUSE=$WHEELHOUSE $TARGET_DEVSTACK_DIR/tools/build_wheels.sh
fi
# Set up Screen
# =============