Fix - Job robustness
- Fix bash conditional error causing the export key job to fail for any chart upgrade - Enhance the image import/configuration job to be more robust with additional retrying and better failure detection Change-Id: I874a98587a56b03905d740cd53cdd70a8419a04e
This commit is contained in:
parent
1df0ad3c6d
commit
fced4bd1d2
|
@ -56,11 +56,11 @@ if [ "x$KEY" != "x" ]; then
|
||||||
}
|
}
|
||||||
EOS
|
EOS
|
||||||
while true; do
|
while true; do
|
||||||
result=$(post_secret)
|
export result=$(post_secret)
|
||||||
if [ ! -z "$(echo $result | grep 201)" ]; then
|
if [ ! -z "$(echo "$result" | grep -i '201 Created')" ]; then
|
||||||
echo 'Secret created'
|
echo 'Secret created'
|
||||||
break
|
break
|
||||||
elif [ ! -z "$(echo $result | grep 409)" ]; then
|
elif [ ! -z "$(echo "$result" | grep -i '409 Conflict')" ]; then
|
||||||
echo 'Secret exists, clearing before trying again'
|
echo 'Secret exists, clearing before trying again'
|
||||||
clear_secret
|
clear_secret
|
||||||
else
|
else
|
||||||
|
|
|
@ -16,22 +16,50 @@
|
||||||
|
|
||||||
set -ex
|
set -ex
|
||||||
|
|
||||||
|
import_tries=0
|
||||||
|
|
||||||
|
TRY_LIMIT=${TRY_LIMIT:-1}
|
||||||
|
JOB_TIMEOUT=${JOB_TIMEOUT:-900}
|
||||||
|
RETRY_TIMER=${RETRY_TIMER:-30}
|
||||||
|
|
||||||
|
function start_import {
|
||||||
|
while [[ ${import_tries} -lt $TRY_LIMIT ]]
|
||||||
|
do
|
||||||
|
import_tries=$(($import_tries + 1))
|
||||||
|
echo "Starting image import try ${import_tries}..."
|
||||||
|
maas ${ADMIN_USERNAME} boot-resources import
|
||||||
|
check_for_download
|
||||||
|
if [[ $? -eq 0 ]]
|
||||||
|
then
|
||||||
|
echo "Image import success!"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
function check_for_download {
|
function check_for_download {
|
||||||
|
|
||||||
while [[ ${JOB_TIMEOUT} -gt 0 ]]; do
|
while [[ ${JOB_TIMEOUT} -gt 0 ]]; do
|
||||||
if maas ${ADMIN_USERNAME} boot-resources is-importing | grep -q 'true';
|
if maas ${ADMIN_USERNAME} boot-resources is-importing | grep -q 'true';
|
||||||
then
|
then
|
||||||
echo -e '\nBoot resources currently importing\n'
|
echo -e '\nBoot resources currently importing\n'
|
||||||
let TIMEOUT-=${RETRY_TIMER}
|
let JOB_TIMEOUT-=${RETRY_TIMER}
|
||||||
sleep ${RETRY_TIMER}
|
sleep ${RETRY_TIMER}
|
||||||
else
|
else
|
||||||
echo 'Boot resources have completed importing'
|
synced_imgs=$(maas ${ADMIN_USERNAME} boot-resources read | grep -B 3 'Synced' | grep 'ubuntu' | wc -l)
|
||||||
# TODO(sthussey) Need to check synced images exist - could be a import failure
|
if [[ $synced_imgs -gt 0 ]]
|
||||||
return 0
|
then
|
||||||
|
echo 'Boot resources have completed importing'
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
echo 'Import failed!'
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
exit 1
|
echo "Timeout waiting for import!"
|
||||||
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
function configure_proxy {
|
function configure_proxy {
|
||||||
|
@ -74,8 +102,11 @@ configure_dns
|
||||||
|
|
||||||
# make call to import images
|
# make call to import images
|
||||||
configure_boot_sources
|
configure_boot_sources
|
||||||
maas ${ADMIN_USERNAME} boot-resources import
|
start_import
|
||||||
# see if we can find > 0 images
|
if [[ $? -eq 0 ]]
|
||||||
sleep ${RETRY_TIMER}
|
then
|
||||||
check_for_download
|
configure_images
|
||||||
configure_images
|
else
|
||||||
|
echo "Image import FAILED!"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
|
@ -65,6 +65,7 @@ rules:
|
||||||
- get
|
- get
|
||||||
- create
|
- create
|
||||||
- update
|
- update
|
||||||
|
- delete
|
||||||
resources:
|
resources:
|
||||||
- secrets
|
- secrets
|
||||||
---
|
---
|
||||||
|
|
|
@ -52,6 +52,8 @@ spec:
|
||||||
key: USERNAME
|
key: USERNAME
|
||||||
- name: RETRY_TIMER
|
- name: RETRY_TIMER
|
||||||
value: {{ .Values.jobs.import_boot_resources.retry_timer | quote }}
|
value: {{ .Values.jobs.import_boot_resources.retry_timer | quote }}
|
||||||
|
- name: TRY_LIMIT
|
||||||
|
value: {{ .Values.jobs.import_boot_resources.try_limit | quote }}
|
||||||
- name: MAAS_ENDPOINT
|
- name: MAAS_ENDPOINT
|
||||||
value: {{ tuple "maas_region_ui" "default" "region_ui" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
|
value: {{ tuple "maas_region_ui" "default" "region_ui" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
|
||||||
- name: MAAS_PROXY_ENABLED
|
- name: MAAS_PROXY_ENABLED
|
||||||
|
|
|
@ -81,6 +81,7 @@ images:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
import_boot_resources:
|
import_boot_resources:
|
||||||
|
try_limit: 1
|
||||||
retry_timer: 10
|
retry_timer: 10
|
||||||
#default timeout: 15 minutes
|
#default timeout: 15 minutes
|
||||||
timeout: 900
|
timeout: 900
|
||||||
|
|
Loading…
Reference in New Issue