Make hard-coded busybox image configurable

The image used to test kubectl logs in the validation scripts was hard
coded and is now configurable.

This also makes the power-up-node.sh gate script more robust by making
it wait for the node to be ready.

Change-Id: I531ca8477ac3575dd4249ab5e991881af290fa52

examples/basic/HostSystem.yaml

@@ -75,4 +75,7 @@ data:
     required:
       docker: docker-engine=1.13.1-0~ubuntu-xenial
       socat: socat=1.7.3.1-1
+  validation:
+    pod_logs:
+      image: busybox:1.28.3
 ...

promenade/schemas/HostSystem.yaml

@@ -131,6 +131,17 @@ data:
         - required
       additionalProperties: false
 
+    validation:
+      type: object
+      properties:
+        pod_logs:
+          type: object
+          properties:
+            image:
+              type: string
+          additionalProperties: false
+      additionalProperties: false
+
   required:
     - images
     - packages

promenade/templates/include/utils.sh

@@ -234,7 +234,7 @@ spec:
     kubernetes.io/hostname: ${NODE}
   containers:
   - name: noisy
-    image: busybox:1.28.3
+    image: {{ config.get_path('HostSystem:validation.pod_logs.image', default='busybox:1.28.3') }}
     imagePullPolicy: IfNotPresent
     command:
     - /bin/echo

tools/g2/lib/kube.sh

@@ -42,3 +42,33 @@ kubectl_wait_for_pod() {
         fi
     done
 }
+
+kubectl_wait_for_node_ready() {
+    set +x
+
+    VIA=${1}
+    NODE_NAME=${2}
+    SEC=${3:-300}
+
+    log Waiting $SEC seconds for $NODE_NAME to be ready.
+
+    NODE_READY_JSONPATH='{.status.conditions[?(@.type=="Ready")].status}'
+
+    end=$(($(date +%s) + $SEC))
+    while true; do
+        if (kubectl_cmd "${VIA}" --request-timeout 10s get nodes $NODE_NAME -o jsonpath="${NODE_READY_JSONPATH}" | grep True) ; then
+            log Node $NODE_NAME is ready.
+            break
+        else
+            now=$(date +%s)
+            if [ $now -gt $end ]; then
+                log Node $NODE_NAME was not ready before timeout.
+                fail
+            fi
+            echo -n .
+            sleep 15
+        fi
+    done
+
+    set -x
+}

tools/g2/manifests/resiliency.json

@@ -102,7 +102,9 @@
       "name": "Power up n2",
       "script": "power-up-node.sh",
       "arguments": [
-        "-n", "n2"
+        "-v", "n0",
+        "-n", "n2",
+        "-w", "120"
       ]
     },
     {

tools/g2/stages/power-up-node.sh

@@ -6,11 +6,19 @@ source "${GATE_UTILS}"
 
 declare -a NODES
 
-while getopts "n:s" opt; do
+WAIT=60
+
+while getopts "n:v:w:s" opt; do
     case "${opt}" in
         n)
             NODES+=("${OPTARG}")
             ;;
+        v)
+            VIA="${OPTARG}"
+            ;;
+        w)
+            WAIT="${OPTARG}"
+            ;;
         *)
             echo "Unknown option"
             exit 1
@@ -22,3 +30,7 @@ shift $((OPTIND-1))
 for node in "${NODES[@]}"; do
     vm_start "${node}"
 done
+
+for node in "${NODES[@]}"; do
+    kubectl_wait_for_node_ready "${VIA}" "${node}" "${WAIT}"
+done