kubectl_apply() { VIA=${1} FILE=${2} ssh_cmd "${VIA}" "cat ${FILE} | kubectl apply -f -" } kubectl_cmd() { VIA=${1} shift ssh_cmd "${VIA}" kubectl "${@}" } kubectl_wait_for_pod() { VIA=${1} NAMESPACE=${2} POD_NAME=${3} SEC=${4:-600} log Waiting "${SEC}" seconds for termination of pod "${POD_NAME}" POD_PHASE_JSONPATH='{.status.phase}' end=$(($(date +%s) + SEC)) while true; do POD_PHASE=$(kubectl_cmd "${VIA}" --request-timeout 10s --namespace "${NAMESPACE}" get -o jsonpath="${POD_PHASE_JSONPATH}" pod "${POD_NAME}") if [[ ${POD_PHASE} = "Succeeded" ]]; then log Pod "${POD_NAME}" succeeded. break elif [[ $POD_PHASE = "Failed" ]]; then log Pod "${POD_NAME}" failed. kubectl_cmd "${VIA}" --request-timeout 10s --namespace "${NAMESPACE}" get -o yaml pod "${POD_NAME}" 1>&2 exit 1 else now=$(date +%s) if [[ $now -gt $end ]]; then log Pod did not terminate before timeout. kubectl_cmd "${VIA}" --request-timeout 10s --namespace "${NAMESPACE}" get -o yaml pod "${POD_NAME}" 1>&2 exit 1 fi sleep 1 fi done } kubectl_wait_for_node_ready() { set +x VIA=${1} NODE_NAME=${2} SEC=${3:-300} log Waiting $SEC seconds for $NODE_NAME to be ready. NODE_READY_JSONPATH='{.status.conditions[?(@.type=="Ready")].status}' end=$(($(date +%s) + $SEC)) while true; do if (kubectl_cmd "${VIA}" --request-timeout 10s get nodes $NODE_NAME -o jsonpath="${NODE_READY_JSONPATH}" | grep True) ; then log Node $NODE_NAME is ready. break else now=$(date +%s) if [ $now -gt $end ]; then log Node $NODE_NAME was not ready before timeout. fail fi echo -n . sleep 15 fi done set -x }