(haproxy) Additional config safeguards

- Some reported cases that the haproxy config was corrupted during
  node reboots. Attempt to add additional safeguards of coordination
  between the anchor and the service pod.

- Support nulling out a default entry in the service list

- Add additional log statements in the anchor

Change-Id: Ie673c50e1037d5dff2b9f67b14032e188183a5d9

charts/haproxy/templates/bin/_anchor.tpl

@@ -35,10 +35,11 @@ install_config() {
     SUCCESS=1
     # Inject global and default config
     mkdir -p $(dirname "$HAPROXY_CONF")
-    cp "$HAPROXY_HEADER" "$NEXT_HAPROXY_CONF"
+    cat "$HAPROXY_HEADER" > "$NEXT_HAPROXY_CONF"
 
     {{- range $namespace, $services := $envAll.Values.conf.anchor.services }}
     {{- range $service, $svc_data := $services }}
+    {{- if $svc_data }}
     {{- $fe_count = add $fe_count 1 }}
 
     echo Constructing config for namespace=\"{{ $namespace }}\" service=\"{{ $service }}\"
@@ -52,6 +53,12 @@ install_config() {
         --namespace {{ $namespace }} \
             get endpoints {{ $service }} \
                 -o 'jsonpath={.subsets[0].addresses[*].ip}')
+
+    if [ $? -ne 0]; then
+      echo "Unable to retrieve service IPs for {{ $service }}, will retry configuration render."
+      return 1
+    fi
+
     DEST_PORT=$(kubectl \
         --server "$KUBE_URL" \
         --certificate-authority "$KUBE_CA" \
@@ -59,11 +66,18 @@ install_config() {
         --namespace {{ $namespace }} \
             get endpoints {{ $service }} \
                 -o 'jsonpath={.subsets[0].ports[0].port}')
+
+    if [ $? -ne 0]; then
+      echo "Unable to retrieve service port for {{ $service }}, will retry configuration render."
+      return 1
+    fi
+
     set -x
 
     if [ "x$SERVICE_IPS" != "x" ]; then
         if [ "x$DEST_PORT" != "x" ]; then
             IDENTIFIER=$(echo "{{ $namespace }}-{{ $service }}")
+            echo "Adding $IDENTIFIER to haproxy config"
             # Add frontend config
             echo >> "$NEXT_HAPROXY_CONF"
             echo "frontend ${IDENTIFIER}-fe" >> "$NEXT_HAPROXY_CONF"
@@ -86,6 +100,7 @@ install_config() {
             {{- end }}
 
             for IP in $SERVICE_IPS; do
+                echo "Adding backend $IP:$DEST_PORT"
                 echo "  server s$IP $IP:$DEST_PORT" {{ $svc_data.server_opts | quote }} >> "$NEXT_HAPROXY_CONF"
             done
         else
@@ -98,6 +113,7 @@ install_config() {
     fi
     {{- end }}
     {{- end }}
+    {{- end }}
 
     if [ $SUCCESS = 1 ]; then
         mkdir -p $(dirname "$HAPROXY_CONF")
@@ -115,7 +131,10 @@ install_config() {
         fi
         chmod -R go-rwx $(dirname "$HAPROXY_CONF")
         chown -R $RUNASUSER:$RUNASUSER $(dirname "$HAPROXY_CONF")
+        return 0
     fi
+
+    return 1
 }
 
 validate_config() {
@@ -155,9 +174,9 @@ while true; do
         break
     fi
 
-    install_config
-
-    compare_copy_files
+    if install_config; then
+      compare_copy_files
+    fi
 
     sleep {{ .Values.conf.anchor.period }}
 done

charts/haproxy/templates/etc/_haproxy.yaml.tpl

@@ -38,21 +38,40 @@ spec:
           value: {{ .Values.conf.haproxy.container_config_dir }}/haproxy.cfg
         - name: LIVE_HAPROXY_CONF
           value: /tmp/live_haproxy.cfg
+        - name: STAGE_HAPROXY_CONF
+          value: /tmp/stage_haproxy.cfg
       command:
         - /bin/sh
         - -c
         - |
             set -eux
 
-            while [ ! -s "$HAPROXY_CONF" ]; do
-                echo Waiting for "HAPROXY_CONF"
+            test_conf () {
+              if [ ! -s "$HAPROXY_CONF" ]; then
+                 echo "New proposed config not found at $HAPROXY_CONF"
+                 return 1
+              fi
+              # this is a safety gate to avoid a race of the anchor
+              # changing a possible new config between the validation
+              # and installation
+              echo "Staging proposed config for installation."
+              cp "$HAPROXY_CONF" "$STAGE_HAPROXY_CONF"
+              if [ ! haproxy -c -f "$STAGE_HAPROXY_CONF"]; then
+                echo "Proposed config not valid."
+                return 1
+              fi
+              return 0
+            }
+
+            while ! test_conf; do
                 sleep 1
             done
+
             echo vvv Starting with initial config vvv
-            cat "$HAPROXY_CONF"
+            cat "$STAGE_HAPROXY_CONF"
             echo
-            cp "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"
-            chmod 700 $LIVE_HAPROXY_CONF
+            mv "$STAGE_HAPROXY_CONF" "$LIVE_HAPROXY_CONF"
+            chmod 700 "$LIVE_HAPROXY_CONF"
 
             # NOTE(mark-burnett): sleep for clearer log output
             sleep 1
@@ -64,7 +83,7 @@ spec:
             set +x
             while true; do
                 if ! cmp -s "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"; then
-                    if ! haproxy -c -f "$HAPROXY_CONF"; then
+                    if ! test_conf; then
                       echo New config file appears invalid, refusing to replace.
                     else
                       echo vvv Replacing old config vvv
@@ -72,10 +91,10 @@ spec:
                       echo
 
                       echo vvv With new config vvv
-                      cat "$HAPROXY_CONF"
+                      cat "$STAGE_HAPROXY_CONF"
                       echo
 
-                      cat "$HAPROXY_CONF" > "$LIVE_HAPROXY_CONF"
+                      cp "$STAGE_HAPROXY_CONF" "$LIVE_HAPROXY_CONF"
 
                       # NOTE(mark-burnett): sleep for clearer log output
                       sleep 1