Make devstack nodes jenkins slaves.
Add them as nodes with the tag "devstack-IMAGE" where IMAGE is, eg, oneiric, with a single executor. Change the reap job to delete these jenkins nodes. Change the delete script to simply mark them for deletion. It should then be called by a new job that is a post-build step for the devstack gate job. Another new job should be called as the first build step in the devstack gate, and it should invoke devstack-vm-inprogress.py to disable the node that the job just started running on. In this manner, we end up with single-use jenkins slaves. Change the gate script to expect to be run the host itself, so it no longer needs to ssh and scp/rsync files around. Add a new script, devstack-vm-gate-wrap.sh, which assists running the gate job on a separate devstack host, as is currently done, to test it out without requiring the full Jenkins infrastructure. Change-Id: I28902918406670163d32ae7c2a644055233dc1fa
This commit is contained in:
James E. Blair
parent
b927d0f6a8
commit
f32f06b50b
|
|
@ -26,20 +26,13 @@ import time
|
|||
import vmdatabase
|
||||
import utils
|
||||
|
||||
NODE_ID = sys.argv[1]
|
||||
|
||||
NODE_NAME = sys.argv[1]
|
||||
|
||||
def main():
|
||||
db = vmdatabase.VMDatabase()
|
||||
machine = db.getMachine(NODE_ID)
|
||||
provider = machine.base_image.provider
|
||||
|
||||
client = utils.get_client(provider)
|
||||
|
||||
server = client.servers.get(machine.external_id)
|
||||
utils.delete_server(server)
|
||||
machine.delete()
|
||||
|
||||
machine = db.getMachineByJenkinsName(NODE_NAME)
|
||||
machine.state = vmdatabase.DELETE
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
#!/bin/bash -x
|
||||
|
||||
# Simulate what Jenkins does with the devstack-gate script.
|
||||
|
||||
NODE_IP_ADDR=$1
|
||||
|
||||
cat >$WORKSPACE/test-env.sh <<EOF
|
||||
export WORKSPACE=/home/jenkins/workspace
|
||||
export DEVSTACK_GATE_PREFIX=wip-
|
||||
export SKIP_DEVSTACK_GATE_PROJECT=1
|
||||
export GERRIT_BRANCH=master
|
||||
export GERRIT_PROJECT=testing
|
||||
export JOB_NAME=test
|
||||
export BUILD_NUMBER=42
|
||||
export GERRIT_CHANGE_NUMBER=1234
|
||||
export GERRIT_PATCHSET_NUMBER=1
|
||||
EOF
|
||||
|
||||
rsync -az --delete $WORKSPACE/ $NODE_IP_ADDR:workspace/
|
||||
RETVAL=$?
|
||||
if [ $RETVAL != 0 ]; then
|
||||
exit $RETVAL
|
||||
fi
|
||||
|
||||
rm $WORKSPACE/test-env.sh
|
||||
ssh $NODE_IP_ADDR '. workspace/test-env.sh && workspace/devstack-gate/devstack-vm-gate-wrap.sh'
|
||||
#RETVAL=$?
|
||||
|
|
@ -1,121 +0,0 @@
|
|||
#!/bin/bash -x
|
||||
|
||||
# Script that is run on the devstack vm; configures and
|
||||
# invokes devstack.
|
||||
|
||||
# Copyright (C) 2011-2012 OpenStack LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -o errexit
|
||||
|
||||
DEVSTACK_GATE_TEMPEST=$1
|
||||
|
||||
# Supply specific tests to Tempest in second argument
|
||||
# For example, to execute only the server actions test,
|
||||
# you would supply tempest.test.test_server_actions
|
||||
DEVSTACK_GATE_TEMPEST_TESTS=$2
|
||||
|
||||
# Remove any crontabs left over from the image
|
||||
sudo crontab -u root -r || /bin/true
|
||||
sudo crontab -u jenkins -r || /bin/true
|
||||
|
||||
cd workspace
|
||||
|
||||
DEST=/opt/stack
|
||||
# create the destination directory and ensure it is writable by the user
|
||||
sudo mkdir -p $DEST
|
||||
if [ ! -w $DEST ]; then
|
||||
sudo chown `whoami` $DEST
|
||||
fi
|
||||
|
||||
# Make sure headers for the currently running kernel are installed:
|
||||
sudo apt-get install -y --force-yes linux-headers-`uname -r`
|
||||
|
||||
# Hpcloud provides no swap, but does have a partition mounted at /mnt
|
||||
# we can use:
|
||||
if [ `cat /proc/meminfo | grep SwapTotal | awk '{ print $2; }'` -eq 0 ] &&
|
||||
[ -b /dev/vdb ]; then
|
||||
sudo umount /dev/vdb
|
||||
sudo mkswap /dev/vdb
|
||||
sudo swapon /dev/vdb
|
||||
fi
|
||||
|
||||
# The workspace has been copied over here by devstack-vm-gate.sh
|
||||
mv * /opt/stack
|
||||
cd /opt/stack/devstack
|
||||
|
||||
ENABLED_SERVICES=g-api,g-reg,key,n-api,n-crt,n-obj,n-cpu,n-net,n-vol,n-sch,horizon,mysql,rabbit
|
||||
|
||||
if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then
|
||||
ENABLED_SERVICES=$ENABLED_SERVICES,tempest
|
||||
fi
|
||||
|
||||
cat <<EOF >localrc
|
||||
ACTIVE_TIMEOUT=60
|
||||
BOOT_TIMEOUT=90
|
||||
ASSOCIATE_TIMEOUT=60
|
||||
MYSQL_PASSWORD=secret
|
||||
RABBIT_PASSWORD=secret
|
||||
ADMIN_PASSWORD=secret
|
||||
SERVICE_PASSWORD=secret
|
||||
SERVICE_TOKEN=111222333444
|
||||
ROOTSLEEP=0
|
||||
ENABLED_SERVICES=$ENABLED_SERVICES
|
||||
SKIP_EXERCISES=boot_from_volume,client-env,swift
|
||||
SERVICE_HOST=127.0.0.1
|
||||
SYSLOG=True
|
||||
SCREEN_LOGDIR=/opt/stack/screen-logs
|
||||
FIXED_RANGE=10.1.0.0/24
|
||||
FIXED_NETWORK_SIZE=256
|
||||
EOF
|
||||
|
||||
if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then
|
||||
# We need to disable ratelimiting when running
|
||||
# Tempest tests since so many requests are executed
|
||||
echo "API_RATE_LIMIT=False" >> localrc
|
||||
# Volume tests in Tempest require a number of volumes
|
||||
# to be created, each of 1G size. Devstack's default
|
||||
# volume backing file size is 2G, so we increase to 4G
|
||||
echo "VOLUME_BACKING_FILE_SIZE=4G" >> localrc
|
||||
fi
|
||||
|
||||
# The vm template update job should cache some images in ~/files.
|
||||
# Move them to where devstack expects:
|
||||
if ls ~/cache/files/*; then
|
||||
mv ~/cache/files/* /opt/stack/devstack/files
|
||||
fi
|
||||
|
||||
# Move the PIP cache into position:
|
||||
sudo mkdir -p /var/cache/pip
|
||||
sudo mv ~/cache/pip/* /var/cache/pip
|
||||
|
||||
# Start with a fresh syslog
|
||||
sudo stop rsyslog
|
||||
sudo mv /var/log/syslog /var/log/syslog-pre-devstack
|
||||
sudo touch /var/log/syslog
|
||||
sudo chown /var/log/syslog --ref /var/log/syslog-pre-devstack
|
||||
sudo chmod /var/log/syslog --ref /var/log/syslog-pre-devstack
|
||||
sudo chmod a+r /var/log/syslog
|
||||
sudo start rsyslog
|
||||
|
||||
./stack.sh
|
||||
if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then
|
||||
./tools/configure_tempest.sh
|
||||
cd /opt/stack/tempest
|
||||
nosetests --with-xunit -sv $DEVSTACK_GATE_TEMPEST_TESTS
|
||||
else
|
||||
./exercise.sh
|
||||
fi
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
#!/bin/bash -x
|
||||
|
||||
# Gate commits to several projects on a VM running those projects
|
||||
# configured by devstack.
|
||||
|
||||
# Copyright (C) 2011-2012 OpenStack LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
PROJECTS="openstack-dev/devstack openstack/nova openstack/glance openstack/keystone openstack/python-novaclient openstack/python-keystoneclient openstack/python-quantumclient openstack/python-glanceclient openstack/horizon openstack/tempest"
|
||||
|
||||
# Set to 1 to run the Tempest test suite
|
||||
export DEVSTACK_GATE_TEMPEST=${DEVSTACK_GATE_TEMPEST:-0}
|
||||
|
||||
# Supply specific tests to Tempest in second argument
|
||||
# For example, to execute only the server actions test,
|
||||
# you would supply tempest.test.test_server_actions
|
||||
export DEVSTACK_GATE_TEMPEST_TESTS=${DEVSTACK_GATE_TEMPEST_TESTS:-tempest}
|
||||
|
||||
# Set this variable to skip updating the devstack-gate project itself.
|
||||
# Useful in development so you can edit scripts in place and run them
|
||||
# directly. Do not set in production.
|
||||
# Normally not set, and we do include devstack-gate with the rest of
|
||||
# the projects.
|
||||
if [ -z "$SKIP_DEVSTACK_GATE_PROJECT" ]; then
|
||||
PROJECTS="openstack-ci/devstack-gate $PROJECTS"
|
||||
fi
|
||||
|
||||
# Set this variable to include tempest in the test run.
|
||||
if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then
|
||||
PROJECTS="openstack/tempest $PROJECTS"
|
||||
fi
|
||||
|
||||
cd $WORKSPACE
|
||||
|
||||
for PROJECT in $PROJECTS
|
||||
do
|
||||
echo "Setting up $PROJECT"
|
||||
SHORT_PROJECT=`basename $PROJECT`
|
||||
if [[ ! -e $SHORT_PROJECT ]]; then
|
||||
echo " Need to clone"
|
||||
git clone https://review.openstack.org/p/$PROJECT
|
||||
fi
|
||||
cd $SHORT_PROJECT
|
||||
|
||||
BRANCH=$GERRIT_BRANCH
|
||||
|
||||
# See if this project has this branch, if not, use master
|
||||
git remote update
|
||||
# Ensure that we don't have stale remotes around
|
||||
git remote prune origin
|
||||
if ! git branch -a |grep remotes/origin/$GERRIT_BRANCH>/dev/null; then
|
||||
BRANCH=master
|
||||
fi
|
||||
git reset --hard
|
||||
git clean -x -f -d -q
|
||||
git checkout $BRANCH
|
||||
git reset --hard remotes/origin/$BRANCH
|
||||
git clean -x -f -d -q
|
||||
|
||||
if [[ $GERRIT_PROJECT == $PROJECT ]]; then
|
||||
echo " Merging proposed change"
|
||||
git fetch https://review.openstack.org/p/$PROJECT $GERRIT_REFSPEC
|
||||
git merge FETCH_HEAD
|
||||
else
|
||||
echo " Updating from origin"
|
||||
git pull --ff-only origin $BRANCH
|
||||
fi
|
||||
cd $WORKSPACE
|
||||
done
|
||||
|
||||
# Set GATE_SCRIPT_DIR to point to devstack-gate in the workspace so that
|
||||
# we are testing the proposed change from this point forward.
|
||||
GATE_SCRIPT_DIR=$WORKSPACE/devstack-gate
|
||||
|
||||
# Also, if we're testing devstack-gate, re-exec this script once so
|
||||
# that we can test the new version of it.
|
||||
if [[ $GERRIT_PROJECT == "openstack-ci/devstack-gate" ]] && [[ $RE_EXEC != "true" ]]; then
|
||||
export RE_EXEC="true"
|
||||
exec $GATE_SCRIPT_DIR/devstack-vm-gate-wrap.sh
|
||||
fi
|
||||
|
||||
# Make sure headers for the currently running kernel are installed:
|
||||
sudo apt-get install -y --force-yes linux-headers-`uname -r`
|
||||
|
||||
# Hpcloud provides no swap, but does have a partition mounted at /mnt
|
||||
# we can use:
|
||||
if [ `cat /proc/meminfo | grep SwapTotal | awk '{ print $2; }'` -eq 0 ] &&
|
||||
[ -b /dev/vdb ]; then
|
||||
sudo umount /dev/vdb
|
||||
sudo mkswap /dev/vdb
|
||||
sudo swapon /dev/vdb
|
||||
fi
|
||||
|
||||
# The vm template update job should cache some images in ~/files.
|
||||
# Move them to where devstack expects:
|
||||
if ls ~/cache/files/*; then
|
||||
mv ~/cache/files/* $WORKSPACE/devstack/files
|
||||
fi
|
||||
|
||||
# Move the PIP cache into position:
|
||||
sudo mkdir -p /var/cache/pip
|
||||
sudo mv ~/cache/pip/* /var/cache/pip
|
||||
|
||||
# Start with a fresh syslog
|
||||
sudo stop rsyslog
|
||||
sudo mv /var/log/syslog /var/log/syslog-pre-devstack
|
||||
sudo touch /var/log/syslog
|
||||
sudo chown /var/log/syslog --ref /var/log/syslog-pre-devstack
|
||||
sudo chmod /var/log/syslog --ref /var/log/syslog-pre-devstack
|
||||
sudo chmod a+r /var/log/syslog
|
||||
sudo start rsyslog
|
||||
|
||||
# Run the test
|
||||
$GATE_SCRIPT_DIR/devstack-vm-gate.sh
|
||||
RETVAL=$?
|
||||
|
||||
cd $WORKSPACE
|
||||
# No matter what, archive logs
|
||||
mkdir -p logs
|
||||
rm -f logs/*
|
||||
|
||||
sudo cp /var/log/syslog $WORKSPACE/logs/syslog.txt
|
||||
cp $WORKSPACE/screen-logs/* $WORKSPACE/logs/
|
||||
|
||||
# Make sure jenkins can read all the logs
|
||||
sudo chown -R jenkins.jenkins $WORKSPACE/logs/
|
||||
sudo chmod a+r $WORKSPACE/logs/
|
||||
|
||||
rename 's/\.log$/.txt/' $WORKSPACE/logs/*
|
||||
|
||||
# Remove duplicate logs
|
||||
rm $WORKSPACE/logs/*.*.txt
|
||||
|
||||
exit $RETVAL
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
#!/bin/bash -x
|
||||
|
||||
# Gate commits to several projects on a VM running those projects
|
||||
# configured by devstack.
|
||||
# Script that is run on the devstack vm; configures and
|
||||
# invokes devstack.
|
||||
|
||||
# Copyright (C) 2011-2012 OpenStack LLC.
|
||||
#
|
||||
|
|
@ -19,141 +19,52 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
PROJECTS="openstack-dev/devstack openstack/nova openstack/glance openstack/keystone openstack/python-novaclient openstack/python-keystoneclient openstack/python-quantumclient openstack/python-glanceclient openstack/horizon openstack/tempest"
|
||||
set -o errexit
|
||||
|
||||
# Set to 1 to run the Tempest test suite
|
||||
DEVSTACK_GATE_TEMPEST=${DEVSTACK_GATE_TEMPEST:-0}
|
||||
export DEST=$WORKSPACE
|
||||
|
||||
# Supply specific tests to Tempest in second argument
|
||||
# For example, to execute only the server actions test,
|
||||
# you would supply tempest.test.test_server_actions
|
||||
DEVSTACK_GATE_TEMPEST_TESTS=${DEVSTACK_GATE_TEMPEST_TESTS:-tempest}
|
||||
cd $DEST/devstack
|
||||
|
||||
# Set this variable to skip updating the devstack-gate project itself.
|
||||
# Useful in development so you can edit scripts in place and run them
|
||||
# directly. Do not set in production.
|
||||
# Normally not set, and we do include devstack-gate with the rest of
|
||||
# the projects.
|
||||
if [ -z "$SKIP_DEVSTACK_GATE_PROJECT" ]; then
|
||||
PROJECTS="openstack-ci/devstack-gate $PROJECTS"
|
||||
fi
|
||||
ENABLED_SERVICES=g-api,g-reg,key,n-api,n-crt,n-obj,n-cpu,n-net,n-vol,n-sch,horizon,mysql,rabbit
|
||||
|
||||
# Set this variable to include tempest in the test run.
|
||||
if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then
|
||||
PROJECTS="openstack/tempest $PROJECTS"
|
||||
ENABLED_SERVICES=$ENABLED_SERVICES,tempest
|
||||
fi
|
||||
|
||||
# Set this to 1 to always keep the host around
|
||||
ALWAYS_KEEP=${ALWAYS_KEEP:-0}
|
||||
cat <<EOF >localrc
|
||||
ACTIVE_TIMEOUT=60
|
||||
BOOT_TIMEOUT=90
|
||||
ASSOCIATE_TIMEOUT=60
|
||||
MYSQL_PASSWORD=secret
|
||||
RABBIT_PASSWORD=secret
|
||||
ADMIN_PASSWORD=secret
|
||||
SERVICE_PASSWORD=secret
|
||||
SERVICE_TOKEN=111222333444
|
||||
SWIFT_HASH=1234123412341234
|
||||
ROOTSLEEP=0
|
||||
ENABLED_SERVICES=$ENABLED_SERVICES
|
||||
SKIP_EXERCISES=boot_from_volume,client-env,swift
|
||||
SERVICE_HOST=127.0.0.1
|
||||
SYSLOG=True
|
||||
SCREEN_LOGDIR=$WORKSPACE/screen-logs
|
||||
FIXED_RANGE=10.1.0.0/24
|
||||
FIXED_NETWORK_SIZE=256
|
||||
EOF
|
||||
|
||||
cd $WORKSPACE
|
||||
mkdir -p logs
|
||||
rm -f logs/*
|
||||
|
||||
for PROJECT in $PROJECTS
|
||||
do
|
||||
echo "Setting up $PROJECT"
|
||||
SHORT_PROJECT=`basename $PROJECT`
|
||||
if [[ ! -e $SHORT_PROJECT ]]; then
|
||||
echo " Need to clone"
|
||||
git clone https://review.openstack.org/p/$PROJECT
|
||||
fi
|
||||
cd $SHORT_PROJECT
|
||||
|
||||
BRANCH=$GERRIT_BRANCH
|
||||
|
||||
# See if this project has this branch, if not, use master
|
||||
git remote update
|
||||
# Ensure that we don't have stale remotes around
|
||||
git remote prune origin
|
||||
if ! git branch -a |grep remotes/origin/$GERRIT_BRANCH>/dev/null; then
|
||||
BRANCH=master
|
||||
fi
|
||||
git reset --hard
|
||||
git clean -x -f -d -q
|
||||
git checkout $BRANCH
|
||||
git reset --hard remotes/origin/$BRANCH
|
||||
git clean -x -f -d -q
|
||||
|
||||
if [[ $GERRIT_PROJECT == $PROJECT ]]; then
|
||||
echo " Merging proposed change"
|
||||
git fetch https://review.openstack.org/p/$PROJECT $GERRIT_REFSPEC
|
||||
git merge FETCH_HEAD
|
||||
else
|
||||
echo " Updating from origin"
|
||||
git pull --ff-only origin $BRANCH
|
||||
fi
|
||||
cd $WORKSPACE
|
||||
done
|
||||
|
||||
# Set GATE_SCRIPT_DIR to point to devstack-gate in the workspace so that
|
||||
# we are testing the proposed change from this point forward.
|
||||
GATE_SCRIPT_DIR=$WORKSPACE/devstack-gate
|
||||
|
||||
# Also, if we're testing devstack-gate, re-exec this script once so
|
||||
# that we can test the new version of it.
|
||||
if [[ $GERRIT_PROJECT == "openstack-ci/devstack-gate" ]] && [[ $RE_EXEC != "true" ]]; then
|
||||
export RE_EXEC="true"
|
||||
exec $GATE_SCRIPT_DIR/devstack-vm-gate.sh
|
||||
fi
|
||||
|
||||
$GATE_SCRIPT_DIR/devstack-vm-fetch.py oneiric > node_info.sh || exit $?
|
||||
. node_info.sh
|
||||
|
||||
scp -C $GATE_SCRIPT_DIR/devstack-vm-gate-host.sh $NODE_IP_ADDR:
|
||||
RETVAL=$?
|
||||
if [ $RETVAL != 0 ]; then
|
||||
echo "Recording node run as failure."
|
||||
if [ -n "$RESULT_ID" ]; then
|
||||
$GATE_SCRIPT_DIR/devstack-vm-result.py $RESULT_ID failure
|
||||
fi
|
||||
echo "Deleting host"
|
||||
$GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_ID
|
||||
exit $RETVAL
|
||||
fi
|
||||
|
||||
rsync -az --delete $WORKSPACE/ $NODE_IP_ADDR:workspace/
|
||||
RETVAL=$?
|
||||
if [ $RETVAL != 0 ]; then
|
||||
echo "Recording node run as failure."
|
||||
if [ -n "$RESULT_ID" ]; then
|
||||
$GATE_SCRIPT_DIR/devstack-vm-result.py $RESULT_ID failure
|
||||
fi
|
||||
echo "Deleting host"
|
||||
$GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_ID
|
||||
exit $RETVAL
|
||||
fi
|
||||
|
||||
ssh $NODE_IP_ADDR ./devstack-vm-gate-host.sh $DEVSTACK_GATE_TEMPEST $DEVSTACK_GATE_TEMPEST_TESTS
|
||||
RETVAL=$?
|
||||
# No matter what, archive logs
|
||||
scp -C -q $NODE_IP_ADDR:/var/log/syslog $WORKSPACE/logs/syslog.txt
|
||||
scp -C -q $NODE_IP_ADDR:/opt/stack/screen-logs/* $WORKSPACE/logs/
|
||||
rename 's/\.log$/.txt/' $WORKSPACE/logs/*
|
||||
# Remove duplicate logs
|
||||
rm $WORKSPACE/logs/*.*.txt
|
||||
# Copy XUnit test results from tempest, if run.
|
||||
if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then
|
||||
scp -C -q $NODE_IP_ADDR:/opt/stack/tempest/nosetests.xml $WORKSPACE/tempest/
|
||||
# We need to disable ratelimiting when running
|
||||
# Tempest tests since so many requests are executed
|
||||
echo "API_RATE_LIMIT=False" >> localrc
|
||||
# Volume tests in Tempest require a number of volumes
|
||||
# to be created, each of 1G size. Devstack's default
|
||||
# volume backing file size is 2G, so we increase to 4G
|
||||
echo "VOLUME_BACKING_FILE_SIZE=4G" >> localrc
|
||||
fi
|
||||
|
||||
# Now check whether the run was a success
|
||||
if [ -n "$RESULT_ID" ]; then
|
||||
if [ $RETVAL = 0 ]; then
|
||||
echo "Recording node run as success."
|
||||
$GATE_SCRIPT_DIR/devstack-vm-result.py $RESULT_ID success
|
||||
else
|
||||
echo "Recording node run as failure."
|
||||
$GATE_SCRIPT_DIR/devstack-vm-result.py $RESULT_ID failure
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $RETVAL = 0 ] && [ $ALWAYS_KEEP = 0 ]; then
|
||||
echo "Deleting host"
|
||||
$GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_ID
|
||||
exit $RETVAL
|
||||
else
|
||||
#echo "Giving host to developer"
|
||||
#$GATE_SCRIPT_DIR/devstack-vm-give.py $NODE_ID
|
||||
exit $RETVAL
|
||||
./stack.sh
|
||||
./exercise.sh
|
||||
if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then
|
||||
./tools/configure_tempest.sh
|
||||
cd $DEST/tempest
|
||||
nosetests --with-xunit -sv $DEVSTACK_GATE_TEMPEST_TESTS
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# Remove old devstack VMs that have been given to developers.
|
||||
|
||||
# Copyright (C) 2011-2012 OpenStack LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import getopt
|
||||
import traceback
|
||||
import ConfigParser
|
||||
|
||||
import myjenkins
|
||||
import vmdatabase
|
||||
import utils
|
||||
import novaclient
|
||||
|
||||
NODE_NAME = sys.argv[1]
|
||||
DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
|
||||
os.path.expanduser('~/devstack-gate-secure.conf'))
|
||||
|
||||
|
||||
def main():
|
||||
db = vmdatabase.VMDatabase()
|
||||
|
||||
config=ConfigParser.ConfigParser()
|
||||
config.read(DEVSTACK_GATE_SECURE_CONFIG)
|
||||
|
||||
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
|
||||
config.get('jenkins', 'user'),
|
||||
config.get('jenkins', 'apikey'))
|
||||
jenkins.get_info()
|
||||
|
||||
machine = db.getMachineByJenkinsName(NODE_NAME)
|
||||
machine.state = vmdatabase.USED
|
||||
|
||||
if machine.jenkins_name:
|
||||
if jenkins.node_exists(machine.jenkins_name):
|
||||
jenkins.disable_node(machine.jenkins_name, "Devstack build started")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -25,12 +25,16 @@ import getopt
|
|||
import time
|
||||
import paramiko
|
||||
import traceback
|
||||
import ConfigParser
|
||||
|
||||
import myjenkins
|
||||
import vmdatabase
|
||||
import utils
|
||||
|
||||
PROVIDER_NAME = sys.argv[1]
|
||||
DEVSTACK_GATE_PREFIX = os.environ.get('DEVSTACK_GATE_PREFIX', '')
|
||||
DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
|
||||
os.path.expanduser('~/devstack-gate-secure.conf'))
|
||||
|
||||
ABANDON_TIMEOUT = 900 # assume a machine will never boot if it hasn't
|
||||
# after this amount of time
|
||||
|
|
@ -76,8 +80,21 @@ def launch_node(client, snap_image, image, flavor, last_name):
|
|||
print
|
||||
return server, machine
|
||||
|
||||
def create_jenkins_node(jenkins, machine):
|
||||
name = '%sdevstack-%s-%s-%s' % (DEVSTACK_GATE_PREFIX, machine.base_image.name,
|
||||
machine.base_image.provider.name, machine.id)
|
||||
machine.jenkins_name = name
|
||||
|
||||
def check_machine(client, machine, error_counts):
|
||||
jenkins.create_node(name, numExecutors=1,
|
||||
nodeDescription='Dynamic single use %s slave for devstack' % machine.base_image.name,
|
||||
remoteFS='/home/jenkins',
|
||||
labels='%sdevstack-%s' % (DEVSTACK_GATE_PREFIX, machine.base_image.name),
|
||||
launcher='hudson.plugins.sshslaves.SSHLauncher',
|
||||
launcher_params = {'port': 22, 'username': 'jenkins',
|
||||
'privatekey': '/var/lib/jenkins/.ssh/id_rsa',
|
||||
'host': machine.ip})
|
||||
|
||||
def check_machine(jenkins, client, machine, error_counts):
|
||||
try:
|
||||
server = client.servers.get(machine.external_id)
|
||||
except:
|
||||
|
|
@ -96,6 +113,8 @@ def check_machine(client, machine, error_counts):
|
|||
machine.ip = ip
|
||||
print "Machine %s is running, testing ssh" % machine.id
|
||||
if utils.ssh_connect(ip, 'jenkins'):
|
||||
print "Adding machine %s to Jenkins" % machine.id
|
||||
create_jenkins_node(jenkins, machine)
|
||||
print "Machine %s is ready" % machine.id
|
||||
machine.state = vmdatabase.READY
|
||||
return
|
||||
|
|
@ -116,6 +135,14 @@ def check_machine(client, machine, error_counts):
|
|||
def main():
|
||||
db = vmdatabase.VMDatabase()
|
||||
|
||||
config=ConfigParser.ConfigParser()
|
||||
config.read(DEVSTACK_GATE_SECURE_CONFIG)
|
||||
|
||||
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
|
||||
config.get('jenkins', 'user'),
|
||||
config.get('jenkins', 'apikey'))
|
||||
jenkins.get_info()
|
||||
|
||||
provider = db.getProvider(PROVIDER_NAME)
|
||||
print "Working with provider %s" % provider.name
|
||||
|
||||
|
|
@ -156,7 +183,7 @@ def main():
|
|||
print "Waiting on %s machines" % len(building_machines)
|
||||
for machine in building_machines:
|
||||
try:
|
||||
check_machine(client, machine, error_counts)
|
||||
check_machine(jenkins, client, machine, error_counts)
|
||||
except:
|
||||
traceback.print_exc()
|
||||
print "Abandoning machine %s" % machine.id
|
||||
|
|
|
|||
|
|
@ -23,13 +23,17 @@ import sys
|
|||
import time
|
||||
import getopt
|
||||
import traceback
|
||||
import ConfigParser
|
||||
|
||||
import myjenkins
|
||||
import vmdatabase
|
||||
import utils
|
||||
import novaclient
|
||||
|
||||
PROVIDER_NAME = sys.argv[1]
|
||||
MACHINE_LIFETIME = 24 * 60 * 60 # Amount of time after being used
|
||||
DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG',
|
||||
os.path.expanduser('~/devstack-gate-secure.conf'))
|
||||
|
||||
if '--all-servers' in sys.argv:
|
||||
print "Reaping all known machines"
|
||||
|
|
@ -44,7 +48,7 @@ else:
|
|||
REAP_ALL_IMAGES = False
|
||||
|
||||
|
||||
def delete_machine(client, machine):
|
||||
def delete_machine(jenkins, client, machine):
|
||||
try:
|
||||
server = client.servers.get(machine.external_id)
|
||||
except novaclient.exceptions.NotFound:
|
||||
|
|
@ -54,6 +58,10 @@ def delete_machine(client, machine):
|
|||
if server:
|
||||
utils.delete_server(server)
|
||||
|
||||
if machine.jenkins_name:
|
||||
if jenkins.node_exists(machine.jenkins_name):
|
||||
jenkins.delete_node(machine.jenkins_name)
|
||||
|
||||
machine.delete()
|
||||
|
||||
|
||||
|
|
@ -82,6 +90,14 @@ def delete_image(client, image):
|
|||
def main():
|
||||
db = vmdatabase.VMDatabase()
|
||||
|
||||
config=ConfigParser.ConfigParser()
|
||||
config.read(DEVSTACK_GATE_SECURE_CONFIG)
|
||||
|
||||
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
|
||||
config.get('jenkins', 'user'),
|
||||
config.get('jenkins', 'apikey'))
|
||||
jenkins.get_info()
|
||||
|
||||
print 'Known machines (start):'
|
||||
db.print_state()
|
||||
|
||||
|
|
@ -98,11 +114,12 @@ def main():
|
|||
for machine in provider.machines:
|
||||
# Normally, reap machines that have sat in their current state
|
||||
# for 24 hours, unless that state is READY.
|
||||
if REAP_ALL_SERVERS or (machine.state != vmdatabase.READY and
|
||||
now - machine.state_time > MACHINE_LIFETIME):
|
||||
if (REAP_ALL_SERVERS or (machine.state != vmdatabase.READY and
|
||||
now - machine.state_time > MACHINE_LIFETIME) or
|
||||
machine.state == vmdatabase.DELETE):
|
||||
print 'Deleting machine', machine.name
|
||||
try:
|
||||
delete_machine(client, machine)
|
||||
delete_machine(jenkins, client, machine)
|
||||
except:
|
||||
error = True
|
||||
traceback.print_exc()
|
||||
|
|
@ -136,9 +153,11 @@ def main():
|
|||
continue
|
||||
if machine.state == vmdatabase.BUILDING:
|
||||
continue
|
||||
if machine.state == vmdatabase.HOLD:
|
||||
continue
|
||||
print 'Deleting machine', machine.name
|
||||
try:
|
||||
delete_machine(client, machine)
|
||||
delete_machine(jenkins, client, machine)
|
||||
overcommitment -= 1
|
||||
except:
|
||||
error = True
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ def tokenize(fn, tokens, distribution, comment=None):
|
|||
for line in open(fn):
|
||||
if 'dist:' in line and ('dist:%s' % distribution not in line):
|
||||
continue
|
||||
if 'qpid' in line: continue #XXX
|
||||
if comment and comment in line:
|
||||
line = line[:line.rfind(comment)]
|
||||
line = line.strip()
|
||||
|
|
@ -225,7 +226,8 @@ def build_image(provider, client, base_image, image, flavor, name, branches, tim
|
|||
# We made the snapshot, try deleting the server, but it's okay
|
||||
# if we fail. The reap script will find it and try again.
|
||||
try:
|
||||
utils.delete_server(server)
|
||||
pass #XXX
|
||||
#utils.delete_server(server)
|
||||
except:
|
||||
print "Exception encountered deleting server:"
|
||||
traceback.print_exc()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,93 @@
|
|||
import jenkins
|
||||
import json
|
||||
import urllib
|
||||
import urllib2
|
||||
from jenkins import JenkinsException, NODE_TYPE, CREATE_NODE
|
||||
|
||||
TOGGLE_OFFLINE = '/computer/%(name)s/toggleOffline?offlineMessage=%(msg)s'
|
||||
|
||||
class Jenkins(jenkins.Jenkins):
|
||||
def disable_node(self, name, msg=''):
|
||||
'''
|
||||
Disable a node
|
||||
|
||||
@param name: Jenkins node name
|
||||
@type name: str
|
||||
@param msg: Offline message
|
||||
@type msg: str
|
||||
'''
|
||||
info = self.get_node_info(name)
|
||||
if info['offline']:
|
||||
return
|
||||
self.jenkins_open(urllib2.Request(self.server + TOGGLE_OFFLINE%locals()))
|
||||
|
||||
def enable_node(self, name):
|
||||
'''
|
||||
Enable a node
|
||||
|
||||
@param name: Jenkins node name
|
||||
@type name: str
|
||||
'''
|
||||
info = self.get_node_info(name)
|
||||
if not info['offline']:
|
||||
return
|
||||
msg = ''
|
||||
self.jenkins_open(urllib2.Request(self.server + TOGGLE_OFFLINE%locals()))
|
||||
|
||||
|
||||
def create_node(self, name, numExecutors=2, nodeDescription=None,
|
||||
remoteFS='/var/lib/jenkins', labels=None, exclusive=False,
|
||||
launcher='hudson.slaves.JNLPLauncher', launcher_params={}):
|
||||
'''
|
||||
@param name: name of node to create
|
||||
@type name: str
|
||||
@param numExecutors: number of executors for node
|
||||
@type numExecutors: int
|
||||
@param nodeDescription: Description of node
|
||||
@type nodeDescription: str
|
||||
@param remoteFS: Remote filesystem location to use
|
||||
@type remoteFS: str
|
||||
@param labels: Labels to associate with node
|
||||
@type labels: str
|
||||
@param exclusive: Use this node for tied jobs only
|
||||
@type exclusive: boolean
|
||||
@param launcher: The launch method for the slave
|
||||
@type launcher: str
|
||||
@param launcher_params: Additional parameters for the launcher
|
||||
@type launcher_params: dict
|
||||
'''
|
||||
if self.node_exists(name):
|
||||
raise JenkinsException('node[%s] already exists'%(name))
|
||||
|
||||
mode = 'NORMAL'
|
||||
if exclusive:
|
||||
mode = 'EXCLUSIVE'
|
||||
|
||||
#hudson.plugins.sshslaves.SSHLauncher
|
||||
#hudson.slaves.CommandLauncher
|
||||
#hudson.os.windows.ManagedWindowsServiceLauncher
|
||||
launcher_params['stapler-class'] = launcher
|
||||
|
||||
inner_params = {
|
||||
'name' : name,
|
||||
'nodeDescription' : nodeDescription,
|
||||
'numExecutors' : numExecutors,
|
||||
'remoteFS' : remoteFS,
|
||||
'labelString' : labels,
|
||||
'mode' : mode,
|
||||
'type' : NODE_TYPE,
|
||||
'retentionStrategy' : { 'stapler-class' : 'hudson.slaves.RetentionStrategy$Always' },
|
||||
'nodeProperties' : { 'stapler-class-bag' : 'true' },
|
||||
'launcher' : launcher_params
|
||||
}
|
||||
|
||||
params = {
|
||||
'name' : name,
|
||||
'type' : NODE_TYPE,
|
||||
'json' : json.dumps(inner_params)
|
||||
}
|
||||
|
||||
self.jenkins_open(urllib2.Request(self.server + CREATE_NODE%urllib.urlencode(params)))
|
||||
|
||||
if not self.node_exists(name):
|
||||
raise JenkinsException('create[%s] failed'%(name))
|
||||
|
|
@ -36,6 +36,8 @@ USED = 3
|
|||
ERROR = 4
|
||||
# Keep this machine indefinitely
|
||||
HOLD = 5
|
||||
# Delete this machine immediately (probably a used machine)
|
||||
DELETE = 6
|
||||
|
||||
# Possible Jenkins results
|
||||
RESULT_SUCCESS = 1
|
||||
|
|
@ -84,12 +86,13 @@ snapshot_image_table = Table('snapshot_image', metadata,
|
|||
machine_table = Table('machine', metadata,
|
||||
Column('id', Integer, primary_key=True),
|
||||
Column('base_image_id', Integer, ForeignKey('base_image.id'), index=True, nullable=False),
|
||||
Column('external_id', String(255)), # Provider assigned id for this machine
|
||||
Column('name', String(255)), # Machine name
|
||||
Column('ip', String(255)), # Primary IP address
|
||||
Column('user', String(255)), # Username if ssh keys have been installed, or NULL
|
||||
Column('state', Integer), # One of the above values
|
||||
Column('state_time', Integer), # Time of last state change
|
||||
Column('external_id', String(255)), # Provider assigned id for this machine
|
||||
Column('name', String(255)), # Machine name
|
||||
Column('jenkins_name', String(255)), # Jenkins node name
|
||||
Column('ip', String(255)), # Primary IP address
|
||||
Column('user', String(255)), # Username if ssh keys have been installed, or NULL
|
||||
Column('state', Integer), # One of the above values
|
||||
Column('state_time', Integer), # Time of last state change
|
||||
)
|
||||
result_table = Table('result', metadata,
|
||||
Column('id', Integer, primary_key=True),
|
||||
|
|
@ -359,6 +362,9 @@ class VMDatabase(object):
|
|||
def getMachine(self, id):
|
||||
return self.session.query(Machine).filter_by(id=id)[0]
|
||||
|
||||
def getMachineByJenkinsName(self, name):
|
||||
return self.session.query(Machine).filter_by(jenkins_name=name)[0]
|
||||
|
||||
def getMachineForUse(self, image_name):
|
||||
"""Atomically find a machine that is ready for use, and update
|
||||
its state."""
|
||||
|
|
|
|||
Loading…
Reference in New Issue