Fix - Job robustness

- Fix bash conditional error causing the export
  key job to fail for any chart upgrade
- Enhance the image import/configuration job to
  be more robust with additional retrying and better
  failure detection

Change-Id: I874a98587a56b03905d740cd53cdd70a8419a04e
This commit is contained in:
Scott Hussey 2018-02-27 12:27:46 -06:00 committed by Anthony Lin
parent 1df0ad3c6d
commit fced4bd1d2
5 changed files with 49 additions and 14 deletions

View File

@ -56,11 +56,11 @@ if [ "x$KEY" != "x" ]; then
}
EOS
while true; do
result=$(post_secret)
if [ ! -z "$(echo $result | grep 201)" ]; then
export result=$(post_secret)
if [ ! -z "$(echo "$result" | grep -i '201 Created')" ]; then
echo 'Secret created'
break
elif [ ! -z "$(echo $result | grep 409)" ]; then
elif [ ! -z "$(echo "$result" | grep -i '409 Conflict')" ]; then
echo 'Secret exists, clearing before trying again'
clear_secret
else

View File

@ -16,22 +16,50 @@
set -ex
import_tries=0
TRY_LIMIT=${TRY_LIMIT:-1}
JOB_TIMEOUT=${JOB_TIMEOUT:-900}
RETRY_TIMER=${RETRY_TIMER:-30}
function start_import {
while [[ ${import_tries} -lt $TRY_LIMIT ]]
do
import_tries=$(($import_tries + 1))
echo "Starting image import try ${import_tries}..."
maas ${ADMIN_USERNAME} boot-resources import
check_for_download
if [[ $? -eq 0 ]]
then
echo "Image import success!"
return 0
fi
done
return 1
}
function check_for_download {
while [[ ${JOB_TIMEOUT} -gt 0 ]]; do
if maas ${ADMIN_USERNAME} boot-resources is-importing | grep -q 'true';
then
echo -e '\nBoot resources currently importing\n'
let TIMEOUT-=${RETRY_TIMER}
let JOB_TIMEOUT-=${RETRY_TIMER}
sleep ${RETRY_TIMER}
else
echo 'Boot resources have completed importing'
# TODO(sthussey) Need to check synced images exist - could be a import failure
return 0
synced_imgs=$(maas ${ADMIN_USERNAME} boot-resources read | grep -B 3 'Synced' | grep 'ubuntu' | wc -l)
if [[ $synced_imgs -gt 0 ]]
then
echo 'Boot resources have completed importing'
return 0
else
echo 'Import failed!'
return 1
fi
fi
done
exit 1
echo "Timeout waiting for import!"
return 1
}
function configure_proxy {
@ -74,8 +102,11 @@ configure_dns
# make call to import images
configure_boot_sources
maas ${ADMIN_USERNAME} boot-resources import
# see if we can find > 0 images
sleep ${RETRY_TIMER}
check_for_download
configure_images
start_import
if [[ $? -eq 0 ]]
then
configure_images
else
echo "Image import FAILED!"
exit 1
fi

View File

@ -65,6 +65,7 @@ rules:
- get
- create
- update
- delete
resources:
- secrets
---

View File

@ -52,6 +52,8 @@ spec:
key: USERNAME
- name: RETRY_TIMER
value: {{ .Values.jobs.import_boot_resources.retry_timer | quote }}
- name: TRY_LIMIT
value: {{ .Values.jobs.import_boot_resources.try_limit | quote }}
- name: MAAS_ENDPOINT
value: {{ tuple "maas_region_ui" "default" "region_ui" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
- name: MAAS_PROXY_ENABLED

View File

@ -81,6 +81,7 @@ images:
jobs:
import_boot_resources:
try_limit: 1
retry_timer: 10
#default timeout: 15 minutes
timeout: 900