diff --git a/charts/haproxy/templates/bin/_anchor.tpl b/charts/haproxy/templates/bin/_anchor.tpl index b70c1da6..2da34733 100644 --- a/charts/haproxy/templates/bin/_anchor.tpl +++ b/charts/haproxy/templates/bin/_anchor.tpl @@ -35,10 +35,11 @@ install_config() { SUCCESS=1 # Inject global and default config mkdir -p $(dirname "$HAPROXY_CONF") - cp "$HAPROXY_HEADER" "$NEXT_HAPROXY_CONF" + cat "$HAPROXY_HEADER" > "$NEXT_HAPROXY_CONF" {{- range $namespace, $services := $envAll.Values.conf.anchor.services }} {{- range $service, $svc_data := $services }} + {{- if $svc_data }} {{- $fe_count = add $fe_count 1 }} echo Constructing config for namespace=\"{{ $namespace }}\" service=\"{{ $service }}\" @@ -52,6 +53,12 @@ install_config() { --namespace {{ $namespace }} \ get endpoints {{ $service }} \ -o 'jsonpath={.subsets[0].addresses[*].ip}') + + if [ $? -ne 0]; then + echo "Unable to retrieve service IPs for {{ $service }}, will retry configuration render." + return 1 + fi + DEST_PORT=$(kubectl \ --server "$KUBE_URL" \ --certificate-authority "$KUBE_CA" \ @@ -59,11 +66,18 @@ install_config() { --namespace {{ $namespace }} \ get endpoints {{ $service }} \ -o 'jsonpath={.subsets[0].ports[0].port}') + + if [ $? -ne 0]; then + echo "Unable to retrieve service port for {{ $service }}, will retry configuration render." + return 1 + fi + set -x if [ "x$SERVICE_IPS" != "x" ]; then if [ "x$DEST_PORT" != "x" ]; then IDENTIFIER=$(echo "{{ $namespace }}-{{ $service }}") + echo "Adding $IDENTIFIER to haproxy config" # Add frontend config echo >> "$NEXT_HAPROXY_CONF" echo "frontend ${IDENTIFIER}-fe" >> "$NEXT_HAPROXY_CONF" @@ -86,6 +100,7 @@ install_config() { {{- end }} for IP in $SERVICE_IPS; do + echo "Adding backend $IP:$DEST_PORT" echo " server s$IP $IP:$DEST_PORT" {{ $svc_data.server_opts | quote }} >> "$NEXT_HAPROXY_CONF" done else @@ -98,6 +113,7 @@ install_config() { fi {{- end }} {{- end }} + {{- end }} if [ $SUCCESS = 1 ]; then mkdir -p $(dirname "$HAPROXY_CONF") @@ -115,7 +131,10 @@ install_config() { fi chmod -R go-rwx $(dirname "$HAPROXY_CONF") chown -R $RUNASUSER:$RUNASUSER $(dirname "$HAPROXY_CONF") + return 0 fi + + return 1 } validate_config() { @@ -155,9 +174,9 @@ while true; do break fi - install_config - - compare_copy_files + if install_config; then + compare_copy_files + fi sleep {{ .Values.conf.anchor.period }} done diff --git a/charts/haproxy/templates/etc/_haproxy.yaml.tpl b/charts/haproxy/templates/etc/_haproxy.yaml.tpl index baf20b49..0433a1f7 100644 --- a/charts/haproxy/templates/etc/_haproxy.yaml.tpl +++ b/charts/haproxy/templates/etc/_haproxy.yaml.tpl @@ -38,21 +38,40 @@ spec: value: {{ .Values.conf.haproxy.container_config_dir }}/haproxy.cfg - name: LIVE_HAPROXY_CONF value: /tmp/live_haproxy.cfg + - name: STAGE_HAPROXY_CONF + value: /tmp/stage_haproxy.cfg command: - /bin/sh - -c - | set -eux - while [ ! -s "$HAPROXY_CONF" ]; do - echo Waiting for "HAPROXY_CONF" + test_conf () { + if [ ! -s "$HAPROXY_CONF" ]; then + echo "New proposed config not found at $HAPROXY_CONF" + return 1 + fi + # this is a safety gate to avoid a race of the anchor + # changing a possible new config between the validation + # and installation + echo "Staging proposed config for installation." + cp "$HAPROXY_CONF" "$STAGE_HAPROXY_CONF" + if [ ! haproxy -c -f "$STAGE_HAPROXY_CONF"]; then + echo "Proposed config not valid." + return 1 + fi + return 0 + } + + while ! test_conf; do sleep 1 done + echo vvv Starting with initial config vvv - cat "$HAPROXY_CONF" + cat "$STAGE_HAPROXY_CONF" echo - cp "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF" - chmod 700 $LIVE_HAPROXY_CONF + mv "$STAGE_HAPROXY_CONF" "$LIVE_HAPROXY_CONF" + chmod 700 "$LIVE_HAPROXY_CONF" # NOTE(mark-burnett): sleep for clearer log output sleep 1 @@ -64,7 +83,7 @@ spec: set +x while true; do if ! cmp -s "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"; then - if ! haproxy -c -f "$HAPROXY_CONF"; then + if ! test_conf; then echo New config file appears invalid, refusing to replace. else echo vvv Replacing old config vvv @@ -72,10 +91,10 @@ spec: echo echo vvv With new config vvv - cat "$HAPROXY_CONF" + cat "$STAGE_HAPROXY_CONF" echo - cat "$HAPROXY_CONF" > "$LIVE_HAPROXY_CONF" + cp "$STAGE_HAPROXY_CONF" "$LIVE_HAPROXY_CONF" # NOTE(mark-burnett): sleep for clearer log output sleep 1