Keep logs in $BASE instead of $WORKSPACE

Avoid moving logs into the jenkins homedir unnecessarily, since it
has a tendency to fill up the root filesystem and the SCP publisher
can follow symlinks out of the workspace anyway. We've seen slaves
go offline due to full root filesystems recently, so this should
hopefully reduce the incidence of that problem.

Change-Id: I490ed53989137218f14464708a65969033c108d6
Closes-Bug: #1268732
This commit is contained in:
Jeremy Stanley 2014-01-23 21:52:14 +00:00
parent 227aa46613
commit db80d83877
2 changed files with 61 additions and 54 deletions

View File

@ -241,10 +241,17 @@ else
$WORKSPACE/logs/devstack-gate-setup-workspace-new.txt
fi
# relocate and symlink logs into $BASE to save space on the root filesystem
if [ -d "$WORKSPACE/logs" -a \! -e "$BASE/logs" ]; then
sudo mv $WORKSPACE/logs $BASE/
ln -s $BASE/logs $WORKSPACE/
fi
# Run pre test hook if we have one
if function_exists "pre_test_hook"; then
set -o xtrace
pre_test_hook 2>&1 | tee $WORKSPACE/logs/devstack-gate-pre-test-hook.txt
pre_test_hook 2>&1 | tee $WORKSPACE/devstack-gate-pre-test-hook.txt
sudo mv $WORKSPACE/devstack-gate-pre-test-hook.txt $BASE/logs/
set +o xtrace
fi
@ -256,8 +263,9 @@ RETVAL=$GATE_RETVAL
# Run post test hook if we have one
if [ $GATE_RETVAL -eq 0 ] && function_exists "post_test_hook"; then
set -o xtrace -o pipefail
post_test_hook 2>&1 | tee $WORKSPACE/logs/devstack-gate-post-test-hook.txt
post_test_hook 2>&1 | tee $WORKSPACE/devstack-gate-post-test-hook.txt
RETVAL=$?
sudo mv $WORKSPACE/devstack-gate-post-test-hook.txt $BASE/logs/
set +o xtrace +o pipefail
fi
@ -267,6 +275,7 @@ if [ $GATE_RETVAL -eq 137 ] && [ -f $WORKSPACE/gate.pid ] ; then
sudo kill -s 9 -${GATEPID}
fi
cleanup_host &> $WORKSPACE/logs/devstack-gate-cleanup-host.txt
cleanup_host &> $WORKSPACE/devstack-gate-cleanup-host.txt
sudo mv $WORKSPACE/devstack-gate-cleanup-host.txt $BASE/logs/
exit $RETVAL

View File

@ -341,88 +341,86 @@ function cleanup_host {
# Sleep to give services a chance to flush their log buffers.
sleep 2
sudo cp /var/log/syslog $WORKSPACE/logs/syslog.txt
sudo cp /var/log/kern.log $WORKSPACE/logs/kern_log.txt
sudo cp /var/log/apache2/horizon_error.log $WORKSPACE/logs/horizon_error.log
mkdir $WORKSPACE/logs/rabbitmq/
sudo cp /var/log/rabbitmq/* $WORKSPACE/logs/rabbitmq/
mkdir $WORKSPACE/logs/sudoers.d/
sudo cp /var/log/syslog $BASE/logs/syslog.txt
sudo cp /var/log/kern.log $BASE/logs/kern_log.txt
sudo cp /var/log/apache2/horizon_error.log $BASE/logs/horizon_error.log
sudo mkdir $BASE/logs/rabbitmq/
sudo cp /var/log/rabbitmq/* $BASE/logs/rabbitmq/
sudo mkdir $BASE/logs/sudoers.d/
sudo cp /etc/sudoers.d/* $WORKSPACE/logs/sudoers.d/
sudo cp /etc/sudoers $WORKSPACE/logs/sudoers.txt
sudo cp /etc/sudoers.d/* $BASE/logs/sudoers.d/
sudo cp /etc/sudoers $BASE/logs/sudoers.txt
if [ -d $BASE/old ]; then
mkdir -p $WORKSPACE/logs/old/
mkdir -p $WORKSPACE/logs/new/
mkdir -p $WORKSPACE/logs/grenade/
sudo cp $BASE/old/screen-logs/* $WORKSPACE/logs/old/
sudo cp $BASE/old/devstacklog.txt $WORKSPACE/logs/old/
sudo cp $BASE/old/devstack/localrc $WORKSPACE/logs/old/localrc.txt
sudo cp $BASE/logs/* $WORKSPACE/logs/
sudo cp $BASE/new/grenade/localrc $WORKSPACE/logs/grenade/localrc.txt
NEWLOGTARGET=$WORKSPACE/logs/new
sudo mkdir -p $BASE/logs/old $BASE/logs/new $BASE/logs/grenade
sudo cp $BASE/old/screen-logs/* $BASE/logs/old/
sudo cp $BASE/old/devstacklog.txt $BASE/logs/old/
sudo cp $BASE/old/devstack/localrc $BASE/logs/old/localrc.txt
sudo cp $BASE/new/grenade/localrc $BASE/logs/grenade/localrc.txt
NEWLOGTARGET=$BASE/logs/new
else
NEWLOGTARGET=$WORKSPACE/logs
NEWLOGTARGET=$BASE/logs
fi
sudo cp $BASE/new/screen-logs/* $NEWLOGTARGET/
sudo cp $BASE/new/devstacklog.txt $NEWLOGTARGET/
sudo cp $BASE/new/devstack/localrc $NEWLOGTARGET/localrc.txt
sudo iptables-save > $WORKSPACE/logs/iptables.txt
df -h> $WORKSPACE/logs/df.txt
pip freeze > $WORKSPACE/logs/pip-freeze.txt
sudo iptables-save > $WORKSPACE/iptables.txt
df -h> $WORKSPACE/df.txt
pip freeze > $WORKSPACE/pip-freeze.txt
sudo mv $WORKSPACE/iptables.txt $WORKSPACE/df.txt \
$WORKSPACE/pip-freeze.txt $BASE/logs/
# Process testr artifacts.
if [ -f $BASE/new/tempest/.testrepository/0 ]; then
sudo cp $BASE/new/tempest/.testrepository/0 $WORKSPACE/subunit_log.txt
sudo python /usr/local/jenkins/slave_scripts/subunit2html.py $WORKSPACE/subunit_log.txt $WORKSPACE/testr_results.html
sudo gzip -9 $WORKSPACE/subunit_log.txt
sudo gzip -9 $WORKSPACE/testr_results.html
sudo chown jenkins:jenkins $WORKSPACE/subunit_log.txt.gz $WORKSPACE/testr_results.html.gz
sudo chmod a+r $WORKSPACE/subunit_log.txt.gz $WORKSPACE/testr_results.html.gz
sudo cp $BASE/new/tempest/.testrepository/0 $BASE/logs/subunit_log.txt
sudo python /usr/local/jenkins/slave_scripts/subunit2html.py $BASE/logs/subunit_log.txt $BASE/logs/testr_results.html
sudo gzip -9 $BASE/logs/subunit_log.txt
sudo gzip -9 $BASE/logs/testr_results.html
sudo chown jenkins:jenkins $BASE/logs/subunit_log.txt.gz $BASE/logs/testr_results.html.gz
sudo chmod a+r $BASE/logs/subunit_log.txt.gz $BASE/logs/testr_results.html.gz
elif [ -f $BASE/new/tempest/.testrepository/tmp* ]; then
# If testr timed out, collect temp file from testr
sudo cp $BASE/new/tempest/.testrepository/tmp* $WORKSPACE/subunit_log.txt
sudo gzip -9 $WORKSPACE/subunit_log.txt
sudo chown jenkins:jenkins $WORKSPACE/subunit_log.txt.gz
sudo chmod a+r $WORKSPACE/subunit_log.txt.gz
sudo cp $BASE/new/tempest/.testrepository/tmp* $BASE/logs/subunit_log.txt
sudo gzip -9 $BASE/logs/subunit_log.txt
sudo chown jenkins:jenkins $BASE/logs/subunit_log.txt.gz
sudo chmod a+r $BASE/logs/subunit_log.txt.gz
fi
if [ -f $BASE/new/tempest/tempest.log ] ; then
sudo cp $BASE/new/tempest/tempest.log $WORKSPACE/logs/tempest.log
sudo cp $BASE/new/tempest/tempest.log $BASE/logs/tempest.log
fi
# Make sure jenkins can read all the logs
sudo chown -R jenkins:jenkins $WORKSPACE/logs/
sudo chmod a+r $WORKSPACE/logs/
sudo chown -R jenkins:jenkins $BASE/logs/
sudo chmod a+r $BASE/logs/
rename 's/\.log$/.txt/' $WORKSPACE/logs/*
rename 's/(.*)/$1.txt/' $WORKSPACE/logs/sudoers.d/*
rename 's/\.log$/.txt/' $WORKSPACE/logs/rabbitmq/*
rename 's/\.log$/.txt/' $BASE/logs/*
rename 's/(.*)/$1.txt/' $BASE/logs/sudoers.d/*
rename 's/\.log$/.txt/' $BASE/logs/rabbitmq/*
mv $WORKSPACE/logs/rabbitmq/startup_log \
$WORKSPACE/logs/rabbitmq/startup_log.txt
sudo mv $BASE/logs/rabbitmq/startup_log \
$BASE/logs/rabbitmq/startup_log.txt
# Remove duplicate logs
rm $WORKSPACE/logs/*.*.txt
sudo rm $BASE/logs/*.*.txt
if [ -d $BASE/old ]; then
rename 's/\.log$/.txt/' $WORKSPACE/logs/old/*
rename 's/\.log$/.txt/' $WORKSPACE/logs/new/*
rename 's/\.log$/.txt/' $WORKSPACE/logs/grenade/*
rm $WORKSPACE/logs/old/*.*.txt
rm $WORKSPACE/logs/new/*.*.txt
sudo rename 's/\.log$/.txt/' $BASE/logs/old/*
sudo rename 's/\.log$/.txt/' $BASE/logs/new/*
sudo rename 's/\.log$/.txt/' $BASE/logs/grenade/*
sudo rm $BASE/logs/old/*.*.txt
sudo rm $BASE/logs/new/*.*.txt
fi
# Compress all text logs
find $WORKSPACE/logs -iname '*.txt' -execdir gzip -9 {} \+
find $WORKSPACE/logs -iname '*.dat' -execdir gzip -9 {} \+
sudo find $BASE/logs -iname '*.txt' -execdir gzip -9 {} \+
sudo find $BASE/logs -iname '*.dat' -execdir gzip -9 {} \+
# Save the tempest nosetests results
sudo cp $BASE/new/tempest/nosetests*.xml $WORKSPACE/
sudo chown jenkins:jenkins $WORKSPACE/nosetests*.xml
sudo chmod a+r $WORKSPACE/nosetests*.xml
sudo cp $BASE/new/tempest/nosetests*.xml $BASE/logs/
sudo chown jenkins:jenkins $BASE/logs/nosetests*.xml
sudo chmod a+r $BASE/logs/nosetests*.xml
# Disable detailed logging as we return to the main script
set +o xtrace