#!/usr/bin/env bash ############################################################################### # Trove Stack Builder, the Trove Dev Machine Controller # ############################################################################### # # # This script provides all the functionality to run all the steps from # # setting up the environment, resetting the nova database to running the # # test. # # # ############################################################################### set -x SCRIPT_DIRNAME=$(dirname "$0") PATH_TROVE=${PATH_TROVE:=$(readlink -f "${SCRIPT_DIRNAME}"/../..)} TROVESTACK_SCRIPTS=${TROVESTACK_SCRIPTS:=$(readlink -f "${SCRIPT_DIRNAME}")} TROVESTACK_TESTS=$TROVESTACK_SCRIPTS/../tests/ DEFAULT_LOCAL_CONF=local.conf.rc DEFAULT_LOCALRC=localrc.rc LOCAL_CONF=local.conf LOCALRC=localrc LOCALRC_AUTO=.localrc.auto USER_LOCAL_CONF_NAME=.devstack.$LOCAL_CONF CLOUD_ADMIN_ARG="--os-cloud=devstack-admin" # Make sure we're not affected by the local environment # by unsetting all the 'OS_' variables while read -r ENV_VAR; do unset "${ENV_VAR}"; done < <(env|grep "OS_"|awk -F= '{print $1}') # Now grab the admin credentials from devstack if it's set up. # This is to facilitate setting the ADMIN_PASSWORD correctly # for gate runs. if [ -f $DEST/devstack/accrc/admin/admin ]; then source $DEST/devstack/accrc/admin/admin fi USERHOME=$HOME # Load options not checked into VCS. if [ -f $USERHOME/.trovestack.options.rc ]; then . $USERHOME/.trovestack.options.rc fi if [ -f $TROVESTACK_SCRIPTS/options.rc ]; then . $TROVESTACK_SCRIPTS/options.rc fi # NOTE(mriedem): The gate-trove-functional-dsvm-* job config in project-config # sets this value for Jenkins runs. BRANCH_OVERRIDE=${BRANCH_OVERRIDE:-default} if [[ $BRANCH_OVERRIDE == "default" && $OVERRIDE_ZUUL_BRANCH != "master" ]]; then BRANCH_OVERRIDE=$OVERRIDE_ZUUL_BRANCH fi # Bail on errors. set -e # Get default host ip from interface function get_default_host_ip() { host_iface=$(ip route | grep default | awk '{print $5}' | head -1) echo `LC_ALL=C ip -f inet addr show ${host_iface} | awk '/inet/ {split($2,parts,"/"); print parts[1]}' | head -1` } # Load functions devstack style . $TROVESTACK_SCRIPTS/functions . $TROVESTACK_SCRIPTS/functions_qemu # Pre-set DISTRO and RELEASE variables based on host OS # Can be overridden by env vars DISTRO and RELEASE GetDistro export DISTRO=${DISTRO:-$DISTRO_NAME} export RELEASE=${RELEASE:-$DISTRO_RELEASE} # Load global configuration variables. . $TROVESTACK_SCRIPTS/trovestack.rc . $TROVESTACK_SCRIPTS/reviews.rc # allow overrides from devstack if already set [[ -f $PATH_DEVSTACK_SRC/functions-common ]] && source $PATH_DEVSTACK_SRC/functions-common [[ -f $PATH_DEVSTACK_SRC/functions ]] && source $PATH_DEVSTACK_SRC/functions [[ -f $PATH_DEVSTACK_SRC/lib/apache ]] && source $PATH_DEVSTACK_SRC/lib/apache # Set up variables for the CONF files - this has to happen after loading trovestack.rc, since # TROVE_CONF_DIR is defined there - these will be used by devstack too export TROVE_CONF=$TROVE_CONF_DIR/trove.conf export TROVE_GUESTAGENT_CONF=$TROVE_CONF_DIR/trove-guestagent.conf export TROVE_API_PASTE_INI=$TROVE_CONF_DIR/api-paste.ini export TEST_CONF=$TROVE_CONF_DIR/test.conf # Public facing bits SERVICE_PROTOCOL=${SERVICE_PROTOCOL:-http} NETWORK_INTERFACE=${NETWORK_INTERFACE:-eth0} NETWORK_SUBNET=${NETWORK_SUBNET:-10.0.0.0/24} NETWORK_GATEWAY=${NETWORK_GATEWAY:-10.0.0.1} BRIDGE_IP=${BRIDGE_IP:-172.24.4.1} KEYSTONE_AUTH_HOST=${KEYSTONE_AUTH_HOST:-$SERVICE_HOST} KEYSTONE_AUTH_PROTOCOL=${KEYSTONE_AUTH_PROTOCOL:-$SERVICE_PROTOCOL} KEYSTONE_AUTH_PORT=${KEYSTONE_AUTH_PORT:-35357} GLANCE_HOSTPORT=${GLANCE_HOSTPORT:-$SERVICE_HOST:9292} GLANCE_SERVICE_PROTOCOL=${GLANCE_SERVICE_PROTOCOL:-http} # PATH_TROVE more than likely has file separators, which sed does not like # This will escape them ESCAPED_PATH_TROVE=$(echo $PATH_TROVE | sed 's/\//\\\//g') ESCAPED_TROVESTACK_SCRIPTS=$(echo $TROVESTACK_SCRIPTS | sed 's/\//\\\//g') TROVE_LOGDIR=${TROVE_LOGDIR:-$DEST/logs} TROVE_DEVSTACK_SETTINGS="$DEST/trove/devstack/settings" TROVE_DEVSTACK_PLUGIN="$DEST/trove/devstack/plugin.sh" # DATASTORE_PKG_LOCATION defines the location from where the datastore packages # can be accessed by the DIB elements. This is applicable only for datastores # that do not have a public repository from where their packages can be accessed. # This can either be a url to a private repository or a location on the local # filesystem that contains the datastore packages. DATASTORE_PKG_LOCATION=${DATASTORE_PKG_LOCATION:-} # Support entry points installation of console scripts if [[ -d $PATH_TROVE/bin ]]; then TROVE_BIN_DIR=$PATH_TROVE/bin else TROVE_BIN_DIR=$(get_python_exec_prefix) fi # set up respective package managers if is_fedora; then PKG_INSTALL_OPTS="" PKG_MGR=dnf PKG_GET_ARGS="-y" else PKG_INSTALL_OPTS="DEBIAN_FRONTEND=noninteractive" PKG_MGR=apt-get PKG_GET_ARGS="-y --allow-unauthenticated --force-yes -qq" fi PKG_INSTALL_ARG="install" PKG_UPDATE_ARG="update" ############################################################################### # Utility functions ############################################################################### # Colors that can be used in 'exclaim' COLOR_RED='\033[0;31m' COLOR_GREEN='\033[0;32m' COLOR_BLUE='\033[0;34m' COLOR_NONE='\033[0m' function exclaim () { echo "*******************************************************************************" echo -e "$@" echo "*******************************************************************************" } function pkg_install () { echo Installing $@... sudo -E $PKG_INSTALL_OPTS $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS $PKG_INSTALL_ARG $@ } function pkg_update () { echo Updating $@... sudo -E $PKG_INSTALL_OPTS $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS $PKG_UPDATE_ARG $@ } function set_http_proxy() { if [ ! "${http_proxy}" = '' ]; then HTTP_PROXY="http_proxy=$http_proxy https_proxy=$https_proxy" fi } function get_ip_for_device() { /sbin/ifconfig $1 | awk '/inet addr/{gsub(/addr:/,"");print $2}' } function ip_chunk() { # Given 1-4 returns a bit of where the ip range starts. # Full IP= `ip_chunk 1`.`ip_chunk 2`.`ip_chunk 3`.`ip_chunk 4` get_ip_for_device $1 | cut -d. -f$2 } function dump_env() { # Print out the environment for debug purposes if [[ -n ${TROVESTACK_DUMP_ENV} ]]; then set +e exclaim "Dumping configuration, starting with env vars:" env | sort CLOUDS_YAML=${CLOUDS_YAML:-/etc/openstack/clouds.yaml} for filename in "${TEST_CONF}" "${CLOUDS_YAML}" "${TROVE_CONF}" "${PATH_DEVSTACK_SRC}/${LOCALRC}" "${PATH_DEVSTACK_SRC}/${LOCALRC_AUTO}"; do if [[ -f ${filename} ]]; then exclaim "Dumping contents of '${filename}':" cat ${filename} else exclaim "File '${filename}' not found" fi done exclaim "Dumping pip modules:" pip freeze | sort exclaim "Dumping configuration completed" set -e fi } # Add a flavor and a corresponding flavor.resize # (flavor.resize adds 16 to the memory and one more vcpu) function add_flavor() { local FLAVOR_NAME=$1 local FLAVOR_ID=$2 local FLAVOR_MEMORY_MB=$3 local FLAVOR_ROOT_GB=$4 local FLAVOR_VCPUS=$5 local FLAVOR_SKIP_RESIZE=${6:-""} if [[ -z "$FLAVOR_LIST_FOR_ADD" ]]; then FLAVOR_LIST_FOR_ADD=$(openstack $CLOUD_ADMIN_ARG flavor list | cut -d'|' -f3 | sed -e's/ /,/g') fi base_id=${FLAVOR_ID} base_name_prefix=test ephemeral_name_prefix=${base_name_prefix}.eph for name_prefix in $base_name_prefix $ephemeral_name_prefix; do reg_name=${name_prefix}.${FLAVOR_NAME}-${FLAVOR_ROOT_GB} resize_name=${reg_name}.resize ephemeral=0 if [[ $name_prefix == $ephemeral_name_prefix ]]; then ephemeral=1 fi for name in ${reg_name} ${resize_name}; do id=$base_id memory=${FLAVOR_MEMORY_MB} vcpus=${FLAVOR_VCPUS} if [[ $ephemeral != 0 ]]; then id=${id}e fi if [[ $name == ${resize_name} ]]; then id=${id}r memory=$((${FLAVOR_MEMORY_MB} + 16)) vcpus=$((${FLAVOR_VCPUS} + 1)) fi if [[ $FLAVOR_LIST_FOR_ADD != *",$name,"* ]]; then if [[ -z ${FLAVOR_SKIP_RESIZE} || ${name} == ${reg_name} ]]; then openstack $CLOUD_ADMIN_ARG flavor create $name --id $id --ram $memory --disk $FLAVOR_ROOT_GB --vcpus $vcpus --ephemeral $ephemeral fi fi done done } function get_attribute_id() { openstack --os-cloud=devstack-admin $1 list | grep " $2" | get_field $3 } ############################################################################### # Install all the required dependencies ############################################################################### function install_prep_packages() { # Called before devstack exclaim 'Updating dependencies (part 1a)...' pkg_update exclaim 'Installing dependencies (part 1b)...' pkg_install python-pip if is_fedora; then pkg_install git gettext else #pkg_install git-core kvm-ipxe gettext pkg_install git-core gettext fi } function install_devstack_code() { exclaim "Installing devstack..." # Installs devstack (if needed). if [ ! -d $PATH_DEVSTACK_SRC ]; then echo "DevStack not in a shared folder, cloning from git." mkdir -p $PATH_DEVSTACK_SRC git clone $DEVSTACK_REPO $PATH_DEVSTACK_SRC fi source $PATH_DEVSTACK_SRC/functions-common source $PATH_DEVSTACK_SRC/functions # Switch to a branch if specified. The order the variables are checked is: # DEVSTACK_BRANCH then PROJECT_BRANCH BRANCH_SPECIFIED=$(test -z "${DEVSTACK_BRANCH}${PROJECT_BRANCH}" || echo 'True') if [[ "${BRANCH_SPECIFIED}" = "True" ]]; then PROJ_BRANCH=$(get_project_branch DEVSTACK_BRANCH $PROJECT_BRANCH) ENV_VARS="DEVSTACK_BRANCH' or 'PROJECT_BRANCH" git_checkout "devstack" "$PATH_DEVSTACK_SRC" "$PROJ_BRANCH" "$ENV_VARS" fi exclaim "Installing devstack projects..." # Ensures present user can get to the devstack dirs sudo mkdir -p $PATH_DEVSTACK_OUTPUT if [ ! -w $PATH_DEVSTACK_OUTPUT ]; then sudo chown `whoami` $PATH_DEVSTACK_OUTPUT fi # Clones all of the code to where devstack expects it to be pushd $PATH_DEVSTACK_OUTPUT cmd_clone_projects do_not_force_update $TROVESTACK_SCRIPTS/projects-list \ $TROVESTACK_SCRIPTS/image-projects-list popd } function install_reviews_on_top_of_devstack() { exclaim "Putting gerrit review code on top of the existing devstack code" run_review_for nova $PATH_NOVA $REVIEW_NOVA run_review_for python-novaclient $PATH_PYTHON_NOVACLIENT $REVIEW_PYTHON_NOVACLIENT run_review_for keystone $PATH_KEYSTONE $REVIEW_KEYSTONE run_review_for python-keystoneclient $PATH_KEYSTONECLIENT $REVIEW_PYTHON_KEYSTONECLIENT run_review_for python-openstackclient $PATH_OPENSTACKCLIENT $REVIEW_PYTHON_OPENSTACKCLIENT run_review_for glance $PATH_GLANCE $REVIEW_GLANCE run_review_for swift $PATH_SWIFT $REVIEW_SWIFT run_review_for python-swiftclient $PATH_PYTHON_SWIFTCLIENT $REVIEW_PYTHON_SWIFTCLIENT run_review_for trove $PATH_TROVE $REVIEW_TROVE run_review_for python-troveclient $PATH_PYTHON_TROVECLIENT $REVIEW_PYTHON_TROVECLIENT } function run_review_for() { # Splits based on colon in the REVIEW_ARG and pulls from GIT_NAME=$1 PATH_ARG=$2 REVIEW_ARG=$3 for review in `echo $REVIEW_ARG| tr ":" "\n"` do # This should be the ref spec for what we pull pushd $PATH_ARG git_timed pull https://review.opendev.org/p/openstack/$GIT_NAME refs/changes/$review popd done } function fixup_broken_devstack() { # Nothing to do here, devstack is working : } # Delete all the lines from FILE_NAME between START_TAG and END_TAG # Tags must appear at the beginning of a line function clear_file_lines() { local FILE_NAME=$1 local START_TAG=$2 local END_TAG=$3 sed -i "/^$START_TAG$/,/^$END_TAG$/{/^$START_TAG/!{/^$END_TAG/!d;}}" "$FILE_NAME" } # Checks to see if a variable with the same name as FILE_NAME exists. # Returns 'true' if no varable exists or if the value of the variable # is set to 'true' - returns the VAR_NAME to set otherwise. # FILE_NAME is first converted to uppercase, the extension is removed # and all remaining '.' and spaces are replaced with '_' function check_filename_var() { local FILE_NAME=$1 DEREF_VALUE=false if [ -f "$FILE_NAME" ]; then VAR_NAME=$(basename "$FILE_NAME" ".rc" | tr '[:lower:][:blank:][:punct:]' '[:upper:]__') DEREF_VALUE=$(get_bool "$VAR_NAME" "true") if [ "$DEREF_VALUE" != "true" ]; then DEREF_VALUE=$VAR_NAME fi fi echo "$DEREF_VALUE" } # Add the contents of one file to another, after the given tag # Run through 'eval' if PARSE_FILE is true (defaults to true) # Start with a blank line if BLANK_LINE_TO_START is true (defaults to false) function add_file_contents() { local FILE_NAME=$1 local FILE_TO_ADD=$2 local TAG=$3 local PARSE_FILE=${4:-true} local BLANK_LINE_TO_START=${5:-false} TEMP_FILE=".trovestack.$$" rm -f "$TEMP_FILE" if [ "$BLANK_LINE_TO_START" = "true" ]; then echo "" > "$TEMP_FILE" fi if [ -f "$FILE_TO_ADD" ]; then echo "Adding $FILE_TO_ADD to $FILE_NAME" echo "# Contents from $FILE_TO_ADD" >> "$TEMP_FILE" if [ "$PARSE_FILE" = "true" ]; then eval echo "\"$(cat "$FILE_TO_ADD")\"" >> "$TEMP_FILE" else cat "$FILE_TO_ADD" >> "$TEMP_FILE" fi echo "# End Of Contents from $FILE_TO_ADD" >> "$TEMP_FILE" fi echo "" >> "$TEMP_FILE" sed -i "/^$TAG/r $TEMP_FILE" "$FILE_NAME" rm -f "$TEMP_FILE" } function run_devstack() { exclaim "Running devstack..." # (Re)Creating this lock directory seems sure-fire. rm -rf "$USERHOME/nova_locks" mkdir -p "$USERHOME/nova_locks" TROVE_PRESENT_TAG="# generated-by-trovestack" LOCAL_CONF_D=local.conf.d CONF_MATCH="*.rc" MARKER_TOKEN="#####" USER_LOCAL_CONF=$(readlink -f "${USER_LOCAL_CONF:-$USERHOME/$USER_LOCAL_CONF_NAME}") LOCALRC_OPTS_TAG="$MARKER_TOKEN Trovestack Localrc Options $MARKER_TOKEN" LOCALRC_OPTS_TAG_END="$MARKER_TOKEN End Of Trovestack Localrc Options $MARKER_TOKEN" USER_OPTS_TAG="$MARKER_TOKEN User Specified Options $MARKER_TOKEN" USER_OPTS_TAG_END="$MARKER_TOKEN End Of User Specified Options $MARKER_TOKEN" ADD_OPTS_TAG="$MARKER_TOKEN Additional Options $MARKER_TOKEN" ADD_OPTS_TAG_END="$MARKER_TOKEN End Of Additional Options $MARKER_TOKEN" pushd "$PATH_DEVSTACK_SRC" DEVSTACK_LOCAL_CONF=$LOCAL_CONF # remain backwards compatible with existing localrc files if [ -f "$LOCALRC" ]; then DEVSTACK_LOCAL_CONF=$LOCALRC echo "Old-style devstack config file $PATH_DEVSTACK_SRC/$DEVSTACK_LOCAL_CONF found." echo "Consider removing to generate the preferred-sytle config file $LOCAL_CONF." fi if [ -f "$DEVSTACK_LOCAL_CONF" ]; then # Check if we have already configured the devstack config file already_in_conf=$(grep "$TROVE_PRESENT_TAG" "$DEVSTACK_LOCAL_CONF" | wc -l) if [ "$already_in_conf" == 0 ]; then # We can no longer append to an existing old-style localrc file if [ "$DEVSTACK_LOCAL_CONF" == "$LOCALRC" ]; then echo "The devstack config file $PATH_DEVSTACK_SRC/$DEVSTACK_LOCAL_CONF is too old to append to." echo "Please remove and try again." exit 1 fi # Otherwise append the trovestack version to the existing file eval echo "\"$(cat "$TROVESTACK_SCRIPTS/$DEFAULT_LOCALRC")\"" >> "$DEVSTACK_LOCAL_CONF" fi else # If a devstack config file doesn't exist, create it eval echo "\"$(cat "$TROVESTACK_SCRIPTS/$DEFAULT_LOCAL_CONF")\"" > "$DEVSTACK_LOCAL_CONF" fi # We can only replace sections from the LOCAL_CONF style files if [ "$DEVSTACK_LOCAL_CONF" == "$LOCAL_CONF" ]; then # Clear out all the options clear_file_lines "$DEVSTACK_LOCAL_CONF" "$LOCALRC_OPTS_TAG" "$LOCALRC_OPTS_TAG_END" clear_file_lines "$DEVSTACK_LOCAL_CONF" "$USER_OPTS_TAG" "$USER_OPTS_TAG_END" clear_file_lines "$DEVSTACK_LOCAL_CONF" "$ADD_OPTS_TAG" "$ADD_OPTS_TAG_END" # Add the main localrc file PARSE_FILE="true" BLANK_LINE_TO_START="true" if [ -f "$TROVESTACK_SCRIPTS/$DEFAULT_LOCALRC" ]; then add_file_contents "$DEVSTACK_LOCAL_CONF" "$TROVESTACK_SCRIPTS/$DEFAULT_LOCALRC" "$LOCALRC_OPTS_TAG" "$PARSE_FILE" "$BLANK_LINE_TO_START" fi # Add any user options PARSE_FILE="false" BLANK_LINE_TO_START="true" if [ -f "$USER_LOCAL_CONF" ]; then add_file_contents "$DEVSTACK_LOCAL_CONF" "$USER_LOCAL_CONF" "$USER_OPTS_TAG" "$PARSE_FILE" "$BLANK_LINE_TO_START" fi # Add all the files in the LOCAL_CONF_D directory that match CONF_MATCH (except for sample files) # and that aren't excluded. Files are excluded by having a variable # 'FILENAME_IN_UPPERCASE_MINUS_RC=false' in trovestack.rc # For Example: USING_VAGRANT=false (for the using_vagrant.rc file). PARSE_FILE="true" BLANK_LINE_TO_START="false" while IFS= read -r -d '' CONF_FILE do FILE_NAME_VAR=$(check_filename_var "$CONF_FILE") if [ "$FILE_NAME_VAR" = "true" ]; then add_file_contents "$DEVSTACK_LOCAL_CONF" "$CONF_FILE" "$ADD_OPTS_TAG" "$PARSE_FILE" "$BLANK_LINE_TO_START" else echo "Skipping $CONF_FILE" echo "Use $FILE_NAME_VAR=true to include" fi done < <(find "$TROVESTACK_SCRIPTS/${LOCAL_CONF_D}" -name "${CONF_MATCH}" -follow -not -name "sample*.rc" -type f -print0) # this is to add a blank line for readability add_file_contents "$DEVSTACK_LOCAL_CONF" "" "$ADD_OPTS_TAG" fi ./stack.sh popd } function cmd_install() { sudo mkdir -p $TROVE_LOGDIR # Creates TROVE_LOGDIR if it does not exist if [ ! -w $TROVE_LOGDIR ]; then sudo chown `whoami` $TROVE_LOGDIR fi install_prep_packages install_devstack_code install_reviews_on_top_of_devstack fixup_broken_devstack run_devstack exclaim "${COLOR_GREEN}FINISHED INSTALL${COLOR_NONE}" } ############################################################################### # Build the image # see functions_qemu ############################################################################### # Grab a numbered field from python prettytable output # Fields are numbered starting with 1 # Reverse syntax is supported: -1 is the last field, -2 is second to last, etc. # get_field field-number function get_field() { while read data; do if [ "$1" -lt 0 ]; then field="(\$(NF$1))" else field="\$$(($1 + 1))" fi echo "$data" | awk -F'[ \t]*\\|[ \t]*' "{print $field}" done } function set_bin_path() { if is_fedora; then sed -i "s|%bin_path%|/usr/bin|g" $TEST_CONF else sed -i "s|%bin_path%|/usr/local/bin|g" $TEST_CONF fi } function cmd_set_datastore() { rd_manage datastore_update "$datastore" "" # Use image tags for datastore version. rd_manage datastore_version_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "${DATASTORE_TYPE}" "" "" 1 --image-tags trove rd_manage datastore_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" if [[ -f "$PATH_TROVE"/trove/templates/${DATASTORE_TYPE}/validation-rules.json ]]; then # add the configuration parameters to the database for the kick-start datastore rd_manage db_load_datastore_config_parameters "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "$PATH_TROVE"/trove/templates/${DATASTORE_TYPE}/validation-rules.json fi } ############################################################################### # Run Unit Tests ############################################################################### function cmd_unit_tests() { exclaim "Running Trove Unit Tests..." $PATH_TROVE/run_tests.sh -N } ############################################################################### # Start various OpenStack daemons interactively in a screen session ############################################################################### function cmd_start_deps() { if ! sudo vgs $VOLUME_GROUP; then exclaim "Reconnecting Volume Group to Backing File" sudo losetup -f --show ${VOLUME_BACKING_FILE} fi if ! egrep -q ${SWIFT_DATA_DIR}/drives/sdb1 /proc/mounts; then exclaim "Re-mounting Swift Disk Image" sudo mount -t xfs -o loop,noatime,nodiratime,nobarrier,logbufs=8 ${SWIFT_DISK_IMAGE} ${SWIFT_DATA_DIR}/drives/sdb1 || true fi if [[ -e $PATH_DEVSTACK_SRC/stack-screenrc ]]; then screen -dmS stack -c $PATH_DEVSTACK_SRC/stack-screenrc fi } function cmd_stop_deps() { if screen -ls | grep -q stack; then screen -S stack -X quit rm -f $DEST/status/stack/* fi } ############################################################################### # Initialize Trove ############################################################################### function rd_manage() { pushd $PATH_TROVE $TROVE_BIN_DIR/trove-manage --config-file=$TROVE_CONF "$@" popd } function install_test_packages() { DATASTORE_TYPE=$1 sudo -H $HTTP_PROXY pip install openstack.nose_plugin proboscis pexpect if [[ "$DATASTORE_TYPE" = "couchbase" ]]; then if [[ "$DISTRO" == "ubuntu" ]]; then # Install Couchbase SDK for scenario tests. sudo -H $HTTP_PROXY curl http://packages.couchbase.com/ubuntu/couchbase.key | sudo apt-key add - echo "deb http://packages.couchbase.com/ubuntu trusty trusty/main" | sudo tee /etc/apt/sources.list.d/couchbase-csdk.list sudo -H $HTTP_PROXY apt-get update sudo -H $HTTP_PROXY apt-get --allow-unauthenticated -y install libcouchbase-dev sudo -H $HTTP_PROXY pip install --upgrade couchbase fi fi } function mod_confs() { local DATASTORE_TYPE=$1 local DATASTORE_VERSION=$2 exclaim "Running mod_confs ..." sudo install -b --mode 0664 $TROVESTACK_SCRIPTS/conf/test_begin.conf $TEST_CONF # cmd_dsvm_gate_tests will set this to be $HOME/report TROVE_REPORT_DIR=${TROVE_REPORT_DIR:=$TROVESTACK_SCRIPTS/../report/} EXTRA_CONF=$TROVESTACK_SCRIPTS/conf/test.extra.conf if [[ -e $EXTRA_CONF ]]; then cat $EXTRA_CONF >> $TEST_CONF fi # Append datastore specific configuration file DATASTORE_CONF=$TROVESTACK_SCRIPTS/conf/$DATASTORE_TYPE.conf if [[ ! -f $DATASTORE_CONF ]]; then exclaim "Datastore configuration file ${DATASTORE_CONF} not found" exit 1 fi cat $DATASTORE_CONF | sudo tee -a $TEST_CONF > /dev/null cat $TROVESTACK_SCRIPTS/conf/test_end.conf | sudo tee -a $TEST_CONF > /dev/null #Add the paths to the test conf sed -i "s,%report_directory%,$TROVE_REPORT_DIR,g" $TEST_CONF sed -i "s,%service_host%,$SERVICE_HOST,g" $TEST_CONF # Add the region name into test.conf sed -i "s/%region_name%/${REGION_NAME}/g" $TEST_CONF # Add the tenant id's into test.conf sed -i "s/%service_tenant_id%/$(get_attribute_id project service 1)/g" $TEST_CONF sed -i "s/%alt_demo_tenant_id%/$(get_attribute_id project alt_demo 1)/g" $TEST_CONF sed -i "s/%demo_tenant_id%/$(get_attribute_id project demo 1)/g" $TEST_CONF sed -i "s/%admin_password%/$ADMIN_PASSWORD/g" $TEST_CONF sed -i "s/%service_password%/$SERVICE_PASSWORD/g" $TEST_CONF # Enable neutron tests if needed sed -i "s/%neutron_enabled%/$ENABLE_NEUTRON/g" $TEST_CONF # Enable backup related tests if Swift is enabled sed -i "s/%swift_enabled%/$ENABLE_SWIFT/g" $TEST_CONF # If neutron is enabled, the devstack plugin has already set up the shared # private network for testing. if [[ $ENABLE_NEUTRON = true ]]; then TROVE_NET_ID=$(openstack $CLOUD_ADMIN_ARG network list | grep " $TROVE_PRIVATE_NETWORK_NAME " | awk '{print $2}') TROVE_SUBNET_ID=$(openstack $CLOUD_ADMIN_ARG subnet list | grep " $TROVE_PRIVATE_SUBNET_NAME " | awk '{print $2}') echo "Using network ${TROVE_PRIVATE_NETWORK_NAME} (${TROVE_NET_ID}): ${TROVE_PRIVATE_SUBNET_NAME} (${TROVE_SUBNET_ID})" sed -i "s,%shared_network%,$TROVE_NET_ID,g" $TEST_CONF sed -i "s,%shared_network_subnet%,$TROVE_SUBNET_ID,g" $TEST_CONF else # do not leave invalid keys in the configuration when using Nova for networking sed -i "/%shared_network%/d" $TEST_CONF sed -i "/%shared_network_subnet%/d" $TEST_CONF fi if [[ "$DATASTORE_TYPE" = "vertica" ]]; then # Vertica needs more time than mysql for its boot/start/stop operations. setup_cluster_configs cluster_member_count 3 elif [[ "$DATASTORE_TYPE" = "pxc" ]]; then setup_cluster_configs min_cluster_member_count 2 elif [[ "$DATASTORE_TYPE" = "cassandra" ]]; then setup_cluster_configs cluster_member_count 2 elif [[ "$DATASTORE_TYPE" = "mongodb" ]]; then setup_cluster_configs cluster_member_count 2 # Decrease the number of required config servers per cluster to save resources. iniset $TROVE_CONF $DATASTORE_TYPE num_config_servers_per_cluster 1 fi sed -i "s/%datastore_type%/$DATASTORE_TYPE/g" $TEST_CONF sed -i "s/%datastore_version%/${DATASTORE_VERSION}/g" $TEST_CONF set_bin_path } function setup_cluster_configs() { # Setting cluster_member_count to 2 to decrease cluster spawn time. iniset $TROVE_CONF $DATASTORE_TYPE $1 $2 } # Add useful flavors for testing (with corresponding *.resize flavors) function add_test_flavors() { # name id ram root_vol vcpu # the ram and vcpu for name.resize are automatically calculated # eph and non-eph flavors are created for each entry add_flavor 'tiny' 10 768 4 1 add_flavor 'small' 15 1024 6 1 add_flavor 'small' 16 1024 7 1 add_flavor 'small' 17 1024 8 1 add_flavor 'medium' 20 1536 7 1 add_flavor 'medium' 21 1536 8 1 add_flavor 'large' 25 2048 8 1 add_flavor 'large' 26 2048 13 1 add_flavor 'large' 27 2048 18 1 # This will allow Nova to create an instance, but not enough disk to boot the image add_flavor 'fault_1' 30 1536 1 1 'skip_resize' # This should be enough memory to cause Nova to fail entirely due to too much allocation add_flavor 'fault_2' 31 131072 7 1 'skip_resize' } function cmd_test_init() { local DATASTORE_TYPE=$1 local DATASTORE_VERSION=$2 if [[ -z "${DATASTORE_TYPE}" ]]; then exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}" exit 1 fi exclaim 'Initializing Configuration for Running Tests...' exclaim "Installing python test packages." install_test_packages "${DATASTORE_TYPE}" exclaim "Modifying test.conf and guest.conf with appropriate values." mod_confs "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" exclaim "Creating Test Flavors." add_test_flavors exclaim "Re-installing python-troveclient from git" pip3 uninstall -y python-troveclient pip3 install -U git+https://opendev.org/openstack/python-troveclient@master#egg=python-troveclient } function cmd_install_docker() { exclaim "install and configure docker: $@" # It seems that rocky8 or newer use podman to emulate docker cli. # the daemon.json file may make no sense here for rocky, but it may be useful for centos distro. sudo mkdir /etc/docker sudo tee /etc/docker/daemon.json >/dev/null < /dev/null } function mysql_trove() { echo mysql trove --execute "$@" mysql -u root -p$MYSQL_PASSWORD trove --execute "$@" 2> /dev/null } function cmd_wipe_logs() { for file in `ls $TROVE_LOGDIR/*.log` do echo "Reseting log file $file..." echo "Reset at `date`" > $file done } function cmd_rd_sql() { mysql -u root -p$MYSQL_PASSWORD trove } function cmd_fake_sql() { pushd $PATH_TROVE sqlite3 trove_test.sqlite $@ popd } function cmd_vagrant_ssh() { # Runs a command on a vagrant VM from the host machine. VHOST=`vagrant ssh_config host | awk '/HostName/{print $2}'` VUSER=`vagrant ssh_config host | awk '/User /{print $2}'` VPORT=`vagrant ssh_config host | awk '/Port/{print $2}'` VIDFILE=`vagrant ssh_config host | awk '/IdentityFile/{print $2}'` echo ssh ${VUSER}@${VHOST} -p ${VPORT} -i ${VIDFILE} -o NoHostAuthenticationForLocalhost=yes "$@" ssh ${VUSER}@${VHOST} -p ${VPORT} -i ${VIDFILE} -o NoHostAuthenticationForLocalhost=yes "$@" } function cmd_run_ci() { local DATASTORE_TYPE=$1 local RESTART_TROVE=${2:-$(get_bool RESTART_TROVE "true")} if [ -z "${DATASTORE_TYPE}" ]; then exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}" exit 1 fi exclaim "Running CI suite..." set +e cmd_stop_deps cmd_stop set -e cmd_install cmd_test_init "${DATASTORE_TYPE}" cmd_build_and_upload_image "${DATASTORE_TYPE}" "${RESTART_TROVE}" # Test in fake mode. exclaim "Testing in fake mode." cmd_start_fake FAKE_MODE=True cmd_int_tests cmd_stop # Test in real mode. exclaim "Testing in real mode." cmd_start FAKE_MODE=False cmd_int_tests } function cmd_wipe_queues() { # Obliterate rabbit. for i in stop_app reset start_app "change_password guest $RABBIT_PASSWORD"; \ do sudo rabbitmqctl $i; done } function cmd_clear() { cmd_int_tests --group=dbaas.api.instances.delete clean_instances mysql_nova "DELETE FROM instance_info_caches;" mysql_nova "DELETE FROM instances;" mysql_trove "DELETE FROM instances;" mysql_trove "DELETE FROM service_statuses;" cmd_wipe_queues } function exec_cmd_on_output() { local output_cmd=$1 local exec_cmd=$2 local delete_sleep_time=${3:-0} local skip_pattern=${4:-""} echo "Cleaning up objects from '${output_cmd}'" local skip_cmd="cat" if [[ -n "${skip_pattern}" ]]; then local temp_skip_cmd=(grep -v "${skip_pattern}") skip_cmd=${temp_skip_cmd[*]} fi local max_retry=10 local count=1 local again= while true; do ids=$($output_cmd | ${skip_cmd} | grep -v -e'---' | grep -iv ' id ' | cut -d'|' -f2) if [[ -n $ids ]]; then for id in $ids; do echo -e "Executing: ${exec_cmd} ${id} ${again}" # don't stop if we get an error executing the delete, and don't print # out anything from stderr set +e ${exec_cmd} "${id}" &> /dev/null set -e done sleep "${delete_sleep_time}" else break fi ((count++)) if [[ "$count" -gt "$max_retry" ]]; then exclaim "${COLOR_RED}WARNING: '$output_cmd' still returning output after ${max_retry} delete attempts${COLOR_NONE}" break fi again="${COLOR_BLUE}(again)${COLOR_NONE}" done } function cmd_clean() { local project_name=${1:-alt_demo} exclaim "Cleaning up project '${COLOR_BLUE}${project_name}${COLOR_NONE}'" # reset any stuck backups mysql_trove "update backups set state='COMPLETED'" # clear out any DS version metadata mysql_trove "delete from datastore_version_metadata" # reset any stuck instances, and clear all replicas mysql_trove "update instances set task_id=2, slave_of_id=null" # reset any stuck clusters mysql_trove "update clusters set task_id=1" # get rid of any extraneous quota usage mysql_trove "delete from quota_usages" # mark all instance modules as deleted mysql_trove "update instance_modules set deleted=1" if [[ ! -f "${PATH_DEVSTACK_SRC}"/accrc/${project_name}/admin ]]; then echo "Could not find credentials file for project '${project_name}'" exit 1 fi source "${PATH_DEVSTACK_SRC}"/accrc/${project_name}/admin local cloud_arg=$CLOUD_ADMIN_ARG if [[ $project_name == *"alt"* ]]; then cloud_arg="--os-cloud=devstack-alt-admin" elif [[ $project_name == "demo" ]]; then cloud_arg="--os-cloud=devstack" fi # delete any trove clusters exec_cmd_on_output "trove cluster-list" "trove cluster-delete" 20 # delete any trove instances exec_cmd_on_output "trove list" "trove delete" 10 # delete any backups exec_cmd_on_output "trove backup-list" "trove backup-delete" # clean up any remaining nova instances or cinder volumes exec_cmd_on_output "openstack $cloud_arg server list" "openstack $cloud_arg server delete" 5 exec_cmd_on_output "openstack $cloud_arg volume list" "openstack $cloud_arg volume delete" 1 # delete any config groups since all instances should be gone now exec_cmd_on_output "trove configuration-list" "trove configuration-delete" # delete any modules too exec_cmd_on_output "trove module-list" "trove module-delete" # make sure that security groups are also gone, except the default exec_cmd_on_output "openstack $cloud_arg security group list" "openstack $cloud_arg security group delete" 0 "default" # delete server groups exec_cmd_on_output "openstack $cloud_arg server group list" "openstack $cloud_arg server group delete" } function cmd_kick_start() { local DATASTORE_TYPE=$1 local DATASTORE_VERSION=$2 if [[ -z "${DATASTORE_TYPE}" ]]; then exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}" exit 1 fi exclaim "Running kick-start for $DATASTORE_TYPE" cmd_test_init "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" dump_env } # Start functional test. The guest image should be created and registered in # appropriate datastore before the test, the configuration parameters should # also be loaded as well. DevStack has done all of that. function cmd_gate_tests() { local DATASTORE_TYPE=${1:-'mysql'} local TEST_GROUP=${2:-${DATASTORE_TYPE}} local DATASTORE_VERSION=${3:-'5.7.29'} local HOST_SCP_USERNAME=${4:-$(whoami)} local GUEST_USERNAME=${5:-'ubuntu'} export DATASTORE_TYPE=${DATASTORE_TYPE} export DATASTORE_VERSION=${DATASTORE_VERSION} exclaim "Running cmd_gate_tests ..." export REPORT_DIRECTORY=${REPORT_DIRECTORY:=$HOME/gate-tests-report/} export TROVE_REPORT_DIR=$HOME/gate-tests-report/ export TROVESTACK_DUMP_ENV=true export SSH_DIR=${SSH_DIR:-"$HOME/.ssh"} # The user is used to connect with the db instance during testing. export TROVE_TEST_SSH_USER=${TROVE_TEST_SSH_USER:-"ubuntu"} # This var is used to ssh into the db instance during testing. export TROVE_TEST_SSH_KEY_FILE=${SSH_DIR}/id_rsa cd $TROVESTACK_SCRIPTS # Build and upload guest image, register datastore version. cmd_build_and_upload_image cmd_kick_start "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" cmd_int_tests --group=$TEST_GROUP } function cmd_reset_task() { mysql_trove "UPDATE instances SET task_id=1 WHERE id='$1'" } function cmd_clone_projects() { local UPDATE_PROJECTS=$1 local PROJECT_LIST_FILES=${@:2} for project in $(cat $PROJECT_LIST_FILES); do if [ ! -d $PATH_DEVSTACK_OUTPUT/$project ]; then echo "Creating a new clone of $project..." git_clone $GIT_OPENSTACK/"$project".git ${PATH_DEVSTACK_OUTPUT}/$project master else if [ $UPDATE_PROJECTS != "force_update" ]; then echo "$project was already cloned or exists in a shared folder. Ignoring..." else echo "$project was already cloned. Pulling changes to update." cd $PATH_DEVSTACK_OUTPUT/$project git pull fi fi # Switch to a branch if specified. The order the variables are checked is: # _BRANCH then PROJECT_CLIENT_BRANCH (if a client) then PROJECT_BRANCH # Note: For the Trove project, only TROVE_BRANCH and PYTHON_TROVECLIENT_BRANCH are used PROJECT_BRANCH_NAME=$(eval echo "${project}_BRANCH" | tr '[:lower:]-' '[:upper:]_') PROJECT_BRANCH_VALUE=${!PROJECT_BRANCH_NAME} # TROVE_BRANCH is defaulted to master if not set, so use the original value here if [[ "$project" = "trove" ]]; then PROJECT_BRANCH_VALUE=${TROVE_BRANCH_ORIG} fi BRANCH_SPECIFIED=$(test -z "${PROJECT_BRANCH_VALUE}${PROJECT_CLIENT_BRANCH}${PROJECT_BRANCH}" || echo 'True') if [[ "${BRANCH_SPECIFIED}" = "True" ]]; then # Set up the default branch and env var names for the project DEFAULT_BRANCH="$PROJECT_BRANCH" ENV_VARS="$PROJECT_BRANCH_NAME' or 'PROJECT_BRANCH" # Don't use 'PROJECT_BRANCH' or 'PROJECT_CLIENT_BRANCH' for the Trove project if [[ "$project" =~ "trove" ]]; then DEFAULT_BRANCH=master ENV_VARS="$PROJECT_BRANCH_NAME" # Use 'PROJECT_CLIENT_BRANCH' first for clients elif [[ "$project" =~ "client" ]]; then DEFAULT_BRANCH="${PROJECT_CLIENT_BRANCH:-$PROJECT_BRANCH}" ENV_VARS="$PROJECT_BRANCH_NAME' or 'PROJECT_CLIENT_BRANCH' or 'PROJECT_BRANCH" fi PROJ_BRANCH=$(get_project_branch $PROJECT_BRANCH_NAME $DEFAULT_BRANCH) git_checkout "$project" "$PATH_DEVSTACK_OUTPUT/$project" "$PROJ_BRANCH" "$ENV_VARS" fi done } function cmd_repl() { INT_TEST_OPTIONS=-i cmd_int_tests_white_box --repl --group=_does_not_exist_ $@ } ############################################################################### # Process the user provided command and run the appropriate command ############################################################################### # Let's not run this as the root user if [ $EUID -eq 0 ]; then echo "You are running this script as root. You need to run as a regular user" exit 1 fi # Set this to exit immediately on error set -o errexit set_http_proxy function print_usage() { echo "Usage: $0 [command]" echo " Commands : --setup environment-- install - Install all the required dependencies and bring up tr-api and tr-tmgr - devstack config can be altered by using a USER_LOCAL_CONF file which will be copied into devstack/local.conf on each 'install' run (defaults to \$HOME/$USER_LOCAL_CONF_NAME) - Set DEVSTACK_BRANCH to switch the branch/commit of devstack (i.e. 'stable/kilo' or '7ef2462') test-init - Configure the test configuration files and add keystone test users install-docker - Install docker and configure docker to not manipulate iptables. build-image - Builds the vm image for the trove guest initialize - Reinitialize the trove database, users, services, and test config --helper for environment-- kick-start - kick start the setup of trove. (trovestack test-init/build-image in one step) - Set REBUILD_IMAGE=True to force rebuild (won't use cached image) --trove dependency services-- start-deps - Start or resume daemons Trove depends on. stop-deps - Kill daemons Trove depends on. --trove services-- start - Start or resume Trove daemons. stop - Kill Trove daemons. restart - Runs stop then start for Trove services. --tests-- unit-tests - Run the unit tests.dependencies int-tests - Runs the integration tests (requires all daemons). See trove/tests/int_tests.py for list of registered groups. Examples: Run original MySQL tests: ./trovestack int-tests Run all MySQL scenario tests: ./trovestack int-tests --group=mysql-supported Run single Redis scenario tests: ./trovestack int-tests --group=redis-supported-single Run specific functional tests: ./trovestack int-tests --group=module-create --group=configuration-create simple-tests - Runs the simple integration tests (requires all daemons). dsvm-gate-tests - Configures and runs the int-tests in a devstack vm-gate environment(legacy Zuul v2 jobs only). gate-tests - Configures and runs the int-tests in a devstack vm-gate environment. --tools-- debug - Debug this script (shows all commands). wipe-logs - Resets all log files. rd-sql - Opens the Trove MySQL database. vagrant-ssh - Runs a command from the host on the server. clear - Destroy instances and rabbit queues. clean - Clean up resources created by a failed test run. Takes project_name as an optional parameter (defaults to alt_demo). run - Starts RD but not in a screen. run-fake - Runs the server in fake mode. update-projects - Git pull on all the daemons trove dependencies. reset-task - Sets an instance task to NONE. wipe-queues - Resets RabbitMQ queues. " exit 1 } function run_command() { # Print the available commands if [ $# -lt 1 ]; then print_usage fi case "$1" in "build-image" ) shift; cmd_build_image $@;; "upload-image" ) shift; cmd_build_and_upload_image $@;; "int-tests" ) shift; cmd_int_tests $@;; "install-docker" ) shift; cmd_install_docker $@;; "debug" ) shift; echo "Enabling debugging."; \ set -o xtrace; TROVESTACK_DUMP_ENV=true; run_command $@;; "gate-tests" ) shift; cmd_gate_tests $@;; "wipe-queues" ) shift; cmd_wipe_queues $@;; * ) echo "'$1' not a valid command" exit 1 esac } run_command $@