diff --git a/tools/multi_nodes_gate/airship_gate/lib/ssh.sh b/tools/multi_nodes_gate/airship_gate/lib/ssh.sh index b01a3b08..0b872bf6 100644 --- a/tools/multi_nodes_gate/airship_gate/lib/ssh.sh +++ b/tools/multi_nodes_gate/airship_gate/lib/ssh.sh @@ -2,6 +2,10 @@ rsync_cmd() { rsync -e "ssh -F ${SSH_CONFIG_DIR}/config" "${@}" } +ssh_cmd_raw() { + ssh -F "${SSH_CONFIG_DIR}/config" $@ +} + ssh_cmd() { HOST=${1} shift diff --git a/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json b/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json index 884c952a..75ebac87 100644 --- a/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json +++ b/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json @@ -53,6 +53,11 @@ { "name": "Deploy Site", "script": "shipyard-deploy-site.sh" + }, + { + "name": "Validate Kube", + "script": "validate-kube.sh", + "on_error": "collect_genesis_info.sh" } ], "vm": { diff --git a/tools/multi_nodes_gate/airship_gate/stages/validate-kube.sh b/tools/multi_nodes_gate/airship_gate/stages/validate-kube.sh new file mode 100755 index 00000000..15dc4833 --- /dev/null +++ b/tools/multi_nodes_gate/airship_gate/stages/validate-kube.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +if [[ -n $GATE_DEBUG && $GATE_DEBUG = "1" ]]; then + set -x +fi + +set -e + +# Define NUM_NODES to appropriate values to check this number of k8s nodes. + +function upload_script() { + source "$GATE_UTILS" + BASENAME=$(basename $BASH_SOURCE) + # Copies script to genesis VM + rsync_cmd "$BASH_SOURCE" "$GENESIS_NAME:/root/airship/" + set -o pipefail + ssh_cmd_raw "$GENESIS_NAME" "GATE_DEBUG=${GATE_DEBUG} NUM_NODES=$1 /root/airship/${BASENAME}" 2>&1 | tee -a "$LOG_FILE" + set +o pipefail +} + +function kubectl_retry() { + cnt=0 + while true; do + "$KUBECTL" "$@" + ret=$? + cnt=$((cnt+1)) + if [[ "$ret" -ne "0" ]]; then + if [[ "$cnt" -lt "$MAX_TRIES" ]]; then + sleep $PAUSE + else + return 1 + fi + else + break + fi + done +} + +function check_kube_nodes() { + try=0 + while true; do + nodes_list=$(kubectl_retry get nodes --no-headers) || true + ret=$? + try=$((try+1)) + if [ "$ret" -ne "0" ]; then + if [[ "$try" -lt "$MAX_TRIES" ]]; then + sleep $PAUSE + else + echo -e "Can't get nodes" + return 1 + fi + fi + nodes_list="${nodes_list}\n" + all_nodes=$(echo -en "$nodes_list" | wc -l) + ok_nodes=$(echo -en "$nodes_list" | grep -c -v "NotReady" || true) + if [[ "$all_nodes" == "$1" && "$ok_nodes" == "$1" ]]; then + echo "Nodes: ${all_nodes}" + break + else + echo "Error: not enough nodes(found ${all_nodes}, ready ${ok_nodes}, expected $1)" + return 1 + fi + done +} + +function check_kube_components() { + try=0 + while true; do + res=$(kubectl_retry get cs -o jsonpath="{.items[*].conditions[?(@.type == \"Healthy\")].status}") || true + try=$((try+1)) + + if $(echo $res | grep -q False); then + if [[ "$try" -lt "$MAX_TRIES" ]]; then + sleep $PAUSE + else + echo "Error: kubernetes components are not working properly" + kubectl_retry get cs + exit 1 + fi + else + break + fi + done +} + +if [[ -n "$GATE_UTILS" ]]; then + upload_script $NUM_NODES +else +set +e + KUBECTL="${KUBECTL:-/usr/local/bin/kubectl}" + NUM_NODES="${NUM_NODES:-4}" + PAUSE="${PAUSE:-1}" + MAX_TRIES="${MAX_TRIES:-3}" + if [[ ! -f "$KUBECTL" ]]; then + echo "Error: ${KUBECTL} not found" + exit 1 + fi + exit_code=0 + check_kube_nodes $NUM_NODES + nodes_status=$? + check_kube_components + components_status=$? + if [[ "$nodes_status" -ne "0" || "$components_status" -ne "0" ]]; then + echo "Kubernetes validation failed" + exit 1 + else + echo "Kubernetes validation succeeded" + fi +fi