diff --git a/examples/basic/HostSystem.yaml b/examples/basic/HostSystem.yaml index 8b303875..0dfc485f 100644 --- a/examples/basic/HostSystem.yaml +++ b/examples/basic/HostSystem.yaml @@ -75,4 +75,7 @@ data: required: docker: docker-engine=1.13.1-0~ubuntu-xenial socat: socat=1.7.3.1-1 + validation: + pod_logs: + image: busybox:1.28.3 ... diff --git a/promenade/schemas/HostSystem.yaml b/promenade/schemas/HostSystem.yaml index 3bf35e3f..fe2a9432 100644 --- a/promenade/schemas/HostSystem.yaml +++ b/promenade/schemas/HostSystem.yaml @@ -131,6 +131,17 @@ data: - required additionalProperties: false + validation: + type: object + properties: + pod_logs: + type: object + properties: + image: + type: string + additionalProperties: false + additionalProperties: false + required: - images - packages diff --git a/promenade/templates/include/utils.sh b/promenade/templates/include/utils.sh index 76d4fb3b..8bac561b 100644 --- a/promenade/templates/include/utils.sh +++ b/promenade/templates/include/utils.sh @@ -234,7 +234,7 @@ spec: kubernetes.io/hostname: ${NODE} containers: - name: noisy - image: busybox:1.28.3 + image: {{ config.get_path('HostSystem:validation.pod_logs.image', default='busybox:1.28.3') }} imagePullPolicy: IfNotPresent command: - /bin/echo diff --git a/tools/g2/lib/kube.sh b/tools/g2/lib/kube.sh index acd43331..578bfef2 100644 --- a/tools/g2/lib/kube.sh +++ b/tools/g2/lib/kube.sh @@ -42,3 +42,33 @@ kubectl_wait_for_pod() { fi done } + +kubectl_wait_for_node_ready() { + set +x + + VIA=${1} + NODE_NAME=${2} + SEC=${3:-300} + + log Waiting $SEC seconds for $NODE_NAME to be ready. + + NODE_READY_JSONPATH='{.status.conditions[?(@.type=="Ready")].status}' + + end=$(($(date +%s) + $SEC)) + while true; do + if (kubectl_cmd "${VIA}" --request-timeout 10s get nodes $NODE_NAME -o jsonpath="${NODE_READY_JSONPATH}" | grep True) ; then + log Node $NODE_NAME is ready. + break + else + now=$(date +%s) + if [ $now -gt $end ]; then + log Node $NODE_NAME was not ready before timeout. + fail + fi + echo -n . + sleep 15 + fi + done + + set -x +} diff --git a/tools/g2/manifests/resiliency.json b/tools/g2/manifests/resiliency.json index 151f0bb4..49f7643e 100644 --- a/tools/g2/manifests/resiliency.json +++ b/tools/g2/manifests/resiliency.json @@ -102,7 +102,9 @@ "name": "Power up n2", "script": "power-up-node.sh", "arguments": [ - "-n", "n2" + "-v", "n0", + "-n", "n2", + "-w", "120" ] }, { diff --git a/tools/g2/stages/power-up-node.sh b/tools/g2/stages/power-up-node.sh index 419319c9..2df1134a 100755 --- a/tools/g2/stages/power-up-node.sh +++ b/tools/g2/stages/power-up-node.sh @@ -6,11 +6,19 @@ source "${GATE_UTILS}" declare -a NODES -while getopts "n:s" opt; do +WAIT=60 + +while getopts "n:v:w:s" opt; do case "${opt}" in n) NODES+=("${OPTARG}") ;; + v) + VIA="${OPTARG}" + ;; + w) + WAIT="${OPTARG}" + ;; *) echo "Unknown option" exit 1 @@ -22,3 +30,7 @@ shift $((OPTIND-1)) for node in "${NODES[@]}"; do vm_start "${node}" done + +for node in "${NODES[@]}"; do + kubectl_wait_for_node_ready "${VIA}" "${node}" "${WAIT}" +done