160 lines
7.6 KiB
Bash
Executable File
160 lines
7.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -eux
|
|
set -o pipefail
|
|
|
|
## Signal to toci_gate_test.sh we've started by
|
|
touch /tmp/toci.started
|
|
|
|
exit_value=0
|
|
export CURRENT_DIR=$(dirname ${BASH_SOURCE[0]:-$0})
|
|
export TRIPLEO_CI_DIR=$CURRENT_DIR/../
|
|
|
|
# TODO(sshnaidm): remove this immediately when settings are in yaml files
|
|
source $TRIPLEO_CI_DIR/tripleo-ci/scripts/common_vars.bash
|
|
source $TRIPLEO_CI_DIR/tripleo-ci/scripts/common_functions.sh
|
|
#source $TRIPLEO_CI_DIR/tripleo-ci/scripts/metrics.bash
|
|
|
|
# Clear out any puppet modules on the node placed their by infra configuration
|
|
sudo rm -rf /etc/puppet/modules/*
|
|
|
|
# Copy nodepool public key to jenkins user
|
|
sudo cp /etc/nodepool/id_rsa.pub $HOME/.ssh/
|
|
sudo chown $USER:$USER $HOME/.ssh/id_rsa.pub
|
|
|
|
# TODO(sshnaidm): To create tripleo-ci special yaml config files in oooq
|
|
# for every TOCI_JOBTYPE, i.e. ovb-nonha-ipv6.yml
|
|
if [[ "$TOCI_JOBTYPE" =~ "-ha" ]]; then
|
|
CONFIG=${CONFIG:-"$TRIPLEO_ROOT/tripleo-quickstart/config/general_config/ha.yml"}
|
|
elif [[ "$TOCI_JOBTYPE" =~ "-nonha" ]]; then
|
|
CONFIG=${CONFIG:-"$TRIPLEO_ROOT/tripleo-quickstart/config/general_config/minimal.yml"}
|
|
else
|
|
CONFIG=${CONFIG:-"$TRIPLEO_ROOT/tripleo-quickstart/config/general_config/minimal.yml"}
|
|
fi
|
|
|
|
# Add jenkin user's SSH key to root authorized_keys for Ansible to run
|
|
sudo mkdir -p /root/.ssh/
|
|
sudo cp ${HOME}/.ssh/authorized_keys /root/.ssh/
|
|
sudo chmod 0600 /root/.ssh/authorized_keys
|
|
sudo chown root:root /root/.ssh/authorized_keys
|
|
|
|
# TODO(bkero): Use an ansible role to create this file
|
|
sudo yum install -y qemu-img
|
|
qemu-img create -f qcow2 /home/jenkins/overcloud-full.qcow2 1G
|
|
|
|
# Bootstrap the subnodes
|
|
$TRIPLEO_CI_DIR/tripleo-ci/scripts/tripleo.sh --bootstrap-subnodes 2>&1 | sudo dd of=/var/log/bootstrap-subnodes.log || (tail -n 50 /var/log/bootstrap-subnodes.log && false)
|
|
|
|
# Install our test cert so SSL tests work
|
|
sudo cp $TRIPLEO_ROOT/tripleo-ci/test-environments/overcloud-cacert.pem /etc/pki/ca-trust/source/anchors/
|
|
sudo cp $TRIPLEO_ROOT/tripleo-ci/test-environments/overcloud-cacert-ipv6.pem /etc/pki/ca-trust/source/anchors/
|
|
sudo update-ca-trust extract
|
|
|
|
# TODO(sshnaidm): to move these variables to jobs yaml config files (see above)
|
|
if [[ "${STABLE_RELEASE}" =~ ^(liberty|mitaka)$ ]] ; then
|
|
export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e $TRIPLEO_ROOT/tripleo-ci/test-environments/worker-config-mitaka-and-below.yaml"
|
|
else
|
|
export OVERCLOUD_DEPLOY_ARGS="$OVERCLOUD_DEPLOY_ARGS -e $TRIPLEO_ROOT/tripleo-ci/test-environments/worker-config.yaml -e /usr/share/openstack-tripleo-heat-templates/environments/low-memory-usage.yaml"
|
|
fi
|
|
export OPT_WORKDIR=${WORKSPACE}/.quickstart
|
|
export OOOQ_LOGS=${WORKSPACE}/logs/oooq
|
|
export OOO_WORKDIR_LOCAL=$HOME
|
|
export OOOQ_DEFAULT_ARGS=" --working-dir $OPT_WORKDIR --retain-inventory -T none -e working_dir=$OOO_WORKDIR_LOCAL -R ${STABLE_RELEASE:-master}"
|
|
|
|
export OOOQ_ARGS=" --working-dir ${OPT_WORKDIR} \
|
|
-e working_dir=${OOO_WORKDIR_LOCAL} \
|
|
--bootstrap \
|
|
--no-clone \
|
|
--retain-inventory \
|
|
--tags build,undercloud-setup,undercloud-scripts,undercloud-install,undercloud-post-install,overcloud-scripts,overcloud-deploy \
|
|
--teardown none \
|
|
--release ${STABLE_RELEASE:-master} \
|
|
--config config/general_config/minimal.yml \
|
|
--extra-vars @${TRIPLEO_ROOT}/tripleo-ci/scripts/quickstart/multinode-settings.yml \
|
|
--playbook multinode-playbook.yml \
|
|
--requirements requirements.txt \
|
|
--requirements quickstart-extras-requirements.txt \
|
|
127.0.0.2"
|
|
|
|
#shopt -s extglob
|
|
#rm -rf /opt/stack/new/!(tripleo-ci|tripleo-quickstart|tripleo-quickstart-extras)
|
|
# End of cleaning
|
|
# HINT: If there's no enough space, remove swap file in /root/
|
|
|
|
# TODO(sshnaidm): when collect-logs role will have the same functionality,
|
|
# replace postci function with this role (see in the end of file).
|
|
trap "exit_val=\$?; [ \$exit_val != 0 ] && echo ERROR DURING PREVIOUS COMMAND ^^^ && echo 'See postci.txt in the logs directory for debugging details'; postci \$exit_val 2>&1 | ts '%Y-%m-%d %H:%M:%S.000 |' > $WORKSPACE/logs/postci.log 2>&1" EXIT
|
|
|
|
mkdir -p $WORKSPACE/logs
|
|
|
|
export IP_DEVICE=${IP_DEVICE:-"eth0"}
|
|
MY_IP=$(ip addr show dev $IP_DEVICE | awk '/inet / {gsub("/.*", "") ; print $2}')
|
|
MY_IP_eth1=$(ip addr show dev eth1 | awk '/inet / {gsub("/.*", "") ; print $2}') || MY_IP_eth1=""
|
|
|
|
export http_proxy=""
|
|
undercloud_net_range="192.168.24."
|
|
undercloud_services_ip=$undercloud_net_range"1"
|
|
undercloud_haproxy_public_ip=$undercloud_net_range"2"
|
|
undercloud_haproxy_admin_ip=$undercloud_net_range"3"
|
|
export no_proxy=$undercloud_services_ip,$undercloud_haproxy_public_ip,$undercloud_haproxy_admin_ip,$MY_IP,$MY_IP_eth1
|
|
|
|
[[ ! -e $OPT_WORKDIR ]] && mkdir -p $OPT_WORKDIR && sudo chown -R ${USER}: $OPT_WORKDIR
|
|
sudo mkdir $OOOQ_LOGS && sudo chown -R ${USER}: $OOOQ_LOGS
|
|
|
|
# make the requirements point to local checkout of tripleo-quickstart-extras
|
|
echo "file://${TRIPLEO_ROOT}/tripleo-quickstart-extras/#egg=tripleo-quickstart-extras" > ${TRIPLEO_ROOT}/tripleo-quickstart/quickstart-extras-requirements.txt
|
|
|
|
# TODO(bkero): Correct this with a dynamic inventory configuration
|
|
sed -i 's/^undercloud ansible_host=undercloud/undercloud ansible_host=127.0.0.2/' $TRIPLEO_ROOT/tripleo-ci/scripts/hosts
|
|
|
|
cp $TRIPLEO_ROOT/tripleo-ci/scripts/hosts $OPT_WORKDIR/hosts
|
|
cp $TRIPLEO_ROOT/tripleo-ci/scripts/quickstart/*y*ml $TRIPLEO_ROOT/tripleo-quickstart/playbooks/
|
|
$TRIPLEO_ROOT/tripleo-quickstart/quickstart.sh --install-deps
|
|
|
|
pushd $TRIPLEO_ROOT/tripleo-quickstart/
|
|
# TODO(sshnaidm): fix inventory and collect-logs roles with prepares ssh.config.ansible,
|
|
sed -i '/ansible_user: stack/d' $TRIPLEO_ROOT/tripleo-quickstart/roles/common/defaults/main.yml
|
|
|
|
# We wrap the quickstart call in a timeout, so that we can get logs if the
|
|
# deploy hangs. 90m = 90 minutes = 1.5 hours
|
|
/usr/bin/timeout --preserve-status 90m \
|
|
$TRIPLEO_ROOT/tripleo-quickstart/quickstart.sh \
|
|
${OOOQ_ARGS} 2>&1 | tee $OOOQ_LOGS/quickstart_install.log || exit_value=2
|
|
|
|
# We use the tripleo.sh pingtest rather than the validate-simple role in
|
|
# quickstart-extras so that we can easily support the multinode scenario
|
|
# pingtests. We should add support for that to the validate simple role,
|
|
# and use that instead.
|
|
# https://github.com/openstack/tripleo-heat-templates/tree/master/ci/pingtests
|
|
if [[ -e ${OOO_WORKDIR_LOCAL}/overcloudrc ]]; then
|
|
$TRIPLEO_CI_DIR/tripleo-ci/scripts/tripleo.sh --overcloud-pingtest &>$OOOQ_LOGS/multinode-overcloud-pingtest.txt
|
|
fi
|
|
|
|
sudo journalctl -u os-collect-config | sudo tee /var/log/os-collect-config.txt
|
|
|
|
# TODO(sshnaidm): to prepare collect-logs role for tripleo-ci specific paths
|
|
# and remove this function then
|
|
collect_oooq_logs
|
|
|
|
# TODO(sshnaidm): fix this either in role or quickstart.sh
|
|
# it will not duplicate logs from undercloud and 127.0.0.2
|
|
sed -i 's/hosts: all:!localhost/hosts: all:!localhost:!127.0.0.2/' $OPT_WORKDIR/playbooks/collect-logs.yml ||:
|
|
|
|
$TRIPLEO_ROOT/tripleo-quickstart/quickstart.sh --bootstrap --no-clone \
|
|
$OOOQ_DEFAULT_ARGS \
|
|
--config $CONFIG \
|
|
--playbook collect-logs.yml \
|
|
-e artcl_collect_dir=$OOOQ_LOGS/collected_logs \
|
|
-e @$TRIPLEO_ROOT/tripleo-ci/scripts/quickstart/multinode-settings.yml \
|
|
-e tripleo_root=$TRIPLEO_ROOT \
|
|
127.0.0.2 > $OOOQ_LOGS/quickstart_collectlogs.log || true
|
|
|
|
export ARA_DATABASE="sqlite:///${OPT_WORKDIR}/ara.sqlite"
|
|
$OPT_WORKDIR/bin/ara generate $OOOQ_LOGS/ara || true
|
|
popd
|
|
|
|
echo 'Run completed.'
|
|
# TODO(sshnaidm): remove this when we're sure there's enough space
|
|
# Watch free space, the outage could break jobs
|
|
sudo df -h
|
|
exit $exit_value
|