summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorEgorov, Stanislav (se6518) <segorov@mirantis.com>2019-01-22 11:45:03 -0800
committerEgorov, Stanislav (se6518) <segorov@mirantis.com>2019-01-29 22:41:53 -0800
commit9489f76255e5c2c5337ad86cf4723b0551b54949 (patch)
treeed1eebf79bf7dcf39089310eeeda8e36cf831737
parent1a445f3c0b1467164287b94dc98697be6977e372 (diff)
Script to smoke check kube
1. Export NUM_NODES(by default it will check 4 nodes in cluster). 2. Run as other stages. Change-Id: I31fb525440afe5625445a410824d6a64cb32850e
Notes
Notes (review): Code-Review+1: Drew Walters <drewwalters96@gmail.com> Code-Review+1: Ahmad Mahmoudi <am495p@att.com> Code-Review+2: Scott Hussey <sthussey@att.com> Code-Review+2: Kaspars Skels <kaspars.skels@gmail.com> Workflow+1: Kaspars Skels <kaspars.skels@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Thu, 31 Jan 2019 19:41:42 +0000 Reviewed-on: https://review.openstack.org/632542 Project: openstack/airship-in-a-bottle Branch: refs/heads/master
-rw-r--r--tools/multi_nodes_gate/airship_gate/lib/ssh.sh4
-rw-r--r--tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json5
-rwxr-xr-xtools/multi_nodes_gate/airship_gate/stages/validate-kube.sh121
3 files changed, 130 insertions, 0 deletions
diff --git a/tools/multi_nodes_gate/airship_gate/lib/ssh.sh b/tools/multi_nodes_gate/airship_gate/lib/ssh.sh
index b01a3b0..0b872bf 100644
--- a/tools/multi_nodes_gate/airship_gate/lib/ssh.sh
+++ b/tools/multi_nodes_gate/airship_gate/lib/ssh.sh
@@ -2,6 +2,10 @@ rsync_cmd() {
2 rsync -e "ssh -F ${SSH_CONFIG_DIR}/config" "${@}" 2 rsync -e "ssh -F ${SSH_CONFIG_DIR}/config" "${@}"
3} 3}
4 4
5ssh_cmd_raw() {
6 ssh -F "${SSH_CONFIG_DIR}/config" $@
7}
8
5ssh_cmd() { 9ssh_cmd() {
6 HOST=${1} 10 HOST=${1}
7 shift 11 shift
diff --git a/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json b/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json
index 884c952..75ebac8 100644
--- a/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json
+++ b/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json
@@ -53,6 +53,11 @@
53 { 53 {
54 "name": "Deploy Site", 54 "name": "Deploy Site",
55 "script": "shipyard-deploy-site.sh" 55 "script": "shipyard-deploy-site.sh"
56 },
57 {
58 "name": "Validate Kube",
59 "script": "validate-kube.sh",
60 "on_error": "collect_genesis_info.sh"
56 } 61 }
57 ], 62 ],
58 "vm": { 63 "vm": {
diff --git a/tools/multi_nodes_gate/airship_gate/stages/validate-kube.sh b/tools/multi_nodes_gate/airship_gate/stages/validate-kube.sh
new file mode 100755
index 0000000..15dc483
--- /dev/null
+++ b/tools/multi_nodes_gate/airship_gate/stages/validate-kube.sh
@@ -0,0 +1,121 @@
1#!/usr/bin/env bash
2# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15if [[ -n $GATE_DEBUG && $GATE_DEBUG = "1" ]]; then
16 set -x
17fi
18
19set -e
20
21# Define NUM_NODES to appropriate values to check this number of k8s nodes.
22
23function upload_script() {
24 source "$GATE_UTILS"
25 BASENAME=$(basename $BASH_SOURCE)
26 # Copies script to genesis VM
27 rsync_cmd "$BASH_SOURCE" "$GENESIS_NAME:/root/airship/"
28 set -o pipefail
29 ssh_cmd_raw "$GENESIS_NAME" "GATE_DEBUG=${GATE_DEBUG} NUM_NODES=$1 /root/airship/${BASENAME}" 2>&1 | tee -a "$LOG_FILE"
30 set +o pipefail
31}
32
33function kubectl_retry() {
34 cnt=0
35 while true; do
36 "$KUBECTL" "$@"
37 ret=$?
38 cnt=$((cnt+1))
39 if [[ "$ret" -ne "0" ]]; then
40 if [[ "$cnt" -lt "$MAX_TRIES" ]]; then
41 sleep $PAUSE
42 else
43 return 1
44 fi
45 else
46 break
47 fi
48 done
49}
50
51function check_kube_nodes() {
52 try=0
53 while true; do
54 nodes_list=$(kubectl_retry get nodes --no-headers) || true
55 ret=$?
56 try=$((try+1))
57 if [ "$ret" -ne "0" ]; then
58 if [[ "$try" -lt "$MAX_TRIES" ]]; then
59 sleep $PAUSE
60 else
61 echo -e "Can't get nodes"
62 return 1
63 fi
64 fi
65 nodes_list="${nodes_list}\n"
66 all_nodes=$(echo -en "$nodes_list" | wc -l)
67 ok_nodes=$(echo -en "$nodes_list" | grep -c -v "NotReady" || true)
68 if [[ "$all_nodes" == "$1" && "$ok_nodes" == "$1" ]]; then
69 echo "Nodes: ${all_nodes}"
70 break
71 else
72 echo "Error: not enough nodes(found ${all_nodes}, ready ${ok_nodes}, expected $1)"
73 return 1
74 fi
75 done
76}
77
78function check_kube_components() {
79 try=0
80 while true; do
81 res=$(kubectl_retry get cs -o jsonpath="{.items[*].conditions[?(@.type == \"Healthy\")].status}") || true
82 try=$((try+1))
83
84 if $(echo $res | grep -q False); then
85 if [[ "$try" -lt "$MAX_TRIES" ]]; then
86 sleep $PAUSE
87 else
88 echo "Error: kubernetes components are not working properly"
89 kubectl_retry get cs
90 exit 1
91 fi
92 else
93 break
94 fi
95 done
96}
97
98if [[ -n "$GATE_UTILS" ]]; then
99 upload_script $NUM_NODES
100else
101set +e
102 KUBECTL="${KUBECTL:-/usr/local/bin/kubectl}"
103 NUM_NODES="${NUM_NODES:-4}"
104 PAUSE="${PAUSE:-1}"
105 MAX_TRIES="${MAX_TRIES:-3}"
106 if [[ ! -f "$KUBECTL" ]]; then
107 echo "Error: ${KUBECTL} not found"
108 exit 1
109 fi
110 exit_code=0
111 check_kube_nodes $NUM_NODES
112 nodes_status=$?
113 check_kube_components
114 components_status=$?
115 if [[ "$nodes_status" -ne "0" || "$components_status" -ne "0" ]]; then
116 echo "Kubernetes validation failed"
117 exit 1
118 else
119 echo "Kubernetes validation succeeded"
120 fi
121fi