Script to smoke check kube

1. Export NUM_NODES(by default it will check 4 nodes in cluster).
2. Run as other stages.

Change-Id: I31fb525440afe5625445a410824d6a64cb32850e
This commit is contained in:
Egorov, Stanislav (se6518) 2019-01-22 11:45:03 -08:00
parent 1a445f3c0b
commit 9489f76255
3 changed files with 130 additions and 0 deletions

View File

@ -2,6 +2,10 @@ rsync_cmd() {
rsync -e "ssh -F ${SSH_CONFIG_DIR}/config" "${@}"
}
ssh_cmd_raw() {
ssh -F "${SSH_CONFIG_DIR}/config" $@
}
ssh_cmd() {
HOST=${1}
shift

View File

@ -53,6 +53,11 @@
{
"name": "Deploy Site",
"script": "shipyard-deploy-site.sh"
},
{
"name": "Validate Kube",
"script": "validate-kube.sh",
"on_error": "collect_genesis_info.sh"
}
],
"vm": {

View File

@ -0,0 +1,121 @@
#!/usr/bin/env bash
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [[ -n $GATE_DEBUG && $GATE_DEBUG = "1" ]]; then
set -x
fi
set -e
# Define NUM_NODES to appropriate values to check this number of k8s nodes.
function upload_script() {
source "$GATE_UTILS"
BASENAME=$(basename $BASH_SOURCE)
# Copies script to genesis VM
rsync_cmd "$BASH_SOURCE" "$GENESIS_NAME:/root/airship/"
set -o pipefail
ssh_cmd_raw "$GENESIS_NAME" "GATE_DEBUG=${GATE_DEBUG} NUM_NODES=$1 /root/airship/${BASENAME}" 2>&1 | tee -a "$LOG_FILE"
set +o pipefail
}
function kubectl_retry() {
cnt=0
while true; do
"$KUBECTL" "$@"
ret=$?
cnt=$((cnt+1))
if [[ "$ret" -ne "0" ]]; then
if [[ "$cnt" -lt "$MAX_TRIES" ]]; then
sleep $PAUSE
else
return 1
fi
else
break
fi
done
}
function check_kube_nodes() {
try=0
while true; do
nodes_list=$(kubectl_retry get nodes --no-headers) || true
ret=$?
try=$((try+1))
if [ "$ret" -ne "0" ]; then
if [[ "$try" -lt "$MAX_TRIES" ]]; then
sleep $PAUSE
else
echo -e "Can't get nodes"
return 1
fi
fi
nodes_list="${nodes_list}\n"
all_nodes=$(echo -en "$nodes_list" | wc -l)
ok_nodes=$(echo -en "$nodes_list" | grep -c -v "NotReady" || true)
if [[ "$all_nodes" == "$1" && "$ok_nodes" == "$1" ]]; then
echo "Nodes: ${all_nodes}"
break
else
echo "Error: not enough nodes(found ${all_nodes}, ready ${ok_nodes}, expected $1)"
return 1
fi
done
}
function check_kube_components() {
try=0
while true; do
res=$(kubectl_retry get cs -o jsonpath="{.items[*].conditions[?(@.type == \"Healthy\")].status}") || true
try=$((try+1))
if $(echo $res | grep -q False); then
if [[ "$try" -lt "$MAX_TRIES" ]]; then
sleep $PAUSE
else
echo "Error: kubernetes components are not working properly"
kubectl_retry get cs
exit 1
fi
else
break
fi
done
}
if [[ -n "$GATE_UTILS" ]]; then
upload_script $NUM_NODES
else
set +e
KUBECTL="${KUBECTL:-/usr/local/bin/kubectl}"
NUM_NODES="${NUM_NODES:-4}"
PAUSE="${PAUSE:-1}"
MAX_TRIES="${MAX_TRIES:-3}"
if [[ ! -f "$KUBECTL" ]]; then
echo "Error: ${KUBECTL} not found"
exit 1
fi
exit_code=0
check_kube_nodes $NUM_NODES
nodes_status=$?
check_kube_components
components_status=$?
if [[ "$nodes_status" -ne "0" || "$components_status" -ne "0" ]]; then
echo "Kubernetes validation failed"
exit 1
else
echo "Kubernetes validation succeeded"
fi
fi