diff --git a/jenkins/jobs/wheel-mirror.yaml b/jenkins/jobs/wheel-mirror.yaml index 0d70e585cc..683d833a40 100644 --- a/jenkins/jobs/wheel-mirror.yaml +++ b/jenkins/jobs/wheel-mirror.yaml @@ -65,6 +65,7 @@ python: "{python}" publishers: + - devstack-logs - console-log - job-template: diff --git a/jenkins/scripts/wheel-build.sh b/jenkins/scripts/wheel-build.sh index e3789399c8..d684822b1e 100755 --- a/jenkins/scripts/wheel-build.sh +++ b/jenkins/scripts/wheel-build.sh @@ -5,16 +5,65 @@ WHEELHOUSE_DIR=$1 PROJECT=openstack/requirements WORKING_DIR=`pwd`/$PROJECT PYTHON_VERSION=$2 +LOGS=$WORKSPACE/logs + +FAIL_LOG=${LOGS}/failed.txt + +# preclean logs +mkdir -p ${LOGS} +rm -rf ${LOGS}/* # Extract and iterate over all the branch names. BRANCHES=`git --git-dir=$WORKING_DIR/.git branch -r | grep '^ origin/[^H]'` for BRANCH in $BRANCHES; do git --git-dir=$WORKING_DIR/.git show $BRANCH:upper-constraints.txt \ 2>/dev/null > /tmp/upper-constraints.txt || true + + # setup the building virtualenv. We want to freshen this for each + # branch. rm -rf build_env virtualenv -p $PYTHON_VERSION build_env - for pkg in $(cat /tmp/upper-constraints.txt); do - build_env/bin/pip --log $WORKSPACE/pip.log wheel -w $WHEELHOUSE_DIR "${pkg}" || \ - echo "*** WHEEL BUILD FAILURE: ${pkg}" - done + + # SHORT_BRANCH is just "master","newton","kilo" etc. because this + # keeps the output log hierarchy much simpler. + SHORT_BRANCH=${BRANCH##origin/} + SHORT_BRANCH=${SHORT_BRANCH##stable/} + + # Failed parallel jobs don't fail the whole job, we just report + # the issues for investigation. + set +e + + # This runs all the jobs under "parallel". The stdout, stderr and + # exit status for each pip invocation will be captured into files + # kept in ${LOGS}/build/${SHORT_BRANCH}/1/[package]. The --joblog + # file keeps an overview of all run jobs, which we can probe to + # find failed jobs. + cat /tmp/upper-constraints.txt | \ + parallel --files --progress --joblog ${LOGS}/$SHORT_BRANCH-job.log \ + --results ${LOGS}/build/$SHORT_BRANCH \ + build_env/bin/pip --verbose wheel -w $WHEELHOUSE_DIR {} + set -e + + # Column $7 is the exit status of the job, $14 is the last + # argument to pip, which is our package. + FAILED=$(awk -e '$7!=0 {print $14}' ${LOGS}/$SHORT_BRANCH-job.log) + if [ -n "${FAILED}" ]; then + echo "*** FAILED BUILDS FOR BRANCH ${BRANCH}" >> ${FAIL_LOG} + echo "${FAILED}" >> ${FAIL_LOG} + echo -e "***\n\n" >> ${FAIL_LOG} + fi done + +if [ -f ${FAIL_LOG} ]; then + cat ${FAIL_LOG} +fi + +# XXX This does make a lot of log files; about 80mb after compression. +# In theory we could correlate just the failed logs and keep those +# from the failure logs above. This is currently (2017-01) left as an +# exercise for when the job is stable :) bz2 gave about 20% +# improvement over gzip in testing. +pushd ${LOGS} +tar cvjf build-logs.tar.bz2 ./build +rm -rf ./build +popd