(haproxy) Additional config safeguards
- Some reported cases that the haproxy config was corrupted during node reboots. Attempt to add additional safeguards of coordination between the anchor and the service pod. - Support nulling out a default entry in the service list - Add additional log statements in the anchor Change-Id: Ie673c50e1037d5dff2b9f67b14032e188183a5d9
This commit is contained in:
parent
19169bb458
commit
479d3cc402
|
@ -35,10 +35,11 @@ install_config() {
|
||||||
SUCCESS=1
|
SUCCESS=1
|
||||||
# Inject global and default config
|
# Inject global and default config
|
||||||
mkdir -p $(dirname "$HAPROXY_CONF")
|
mkdir -p $(dirname "$HAPROXY_CONF")
|
||||||
cp "$HAPROXY_HEADER" "$NEXT_HAPROXY_CONF"
|
cat "$HAPROXY_HEADER" > "$NEXT_HAPROXY_CONF"
|
||||||
|
|
||||||
{{- range $namespace, $services := $envAll.Values.conf.anchor.services }}
|
{{- range $namespace, $services := $envAll.Values.conf.anchor.services }}
|
||||||
{{- range $service, $svc_data := $services }}
|
{{- range $service, $svc_data := $services }}
|
||||||
|
{{- if $svc_data }}
|
||||||
{{- $fe_count = add $fe_count 1 }}
|
{{- $fe_count = add $fe_count 1 }}
|
||||||
|
|
||||||
echo Constructing config for namespace=\"{{ $namespace }}\" service=\"{{ $service }}\"
|
echo Constructing config for namespace=\"{{ $namespace }}\" service=\"{{ $service }}\"
|
||||||
|
@ -52,6 +53,12 @@ install_config() {
|
||||||
--namespace {{ $namespace }} \
|
--namespace {{ $namespace }} \
|
||||||
get endpoints {{ $service }} \
|
get endpoints {{ $service }} \
|
||||||
-o 'jsonpath={.subsets[0].addresses[*].ip}')
|
-o 'jsonpath={.subsets[0].addresses[*].ip}')
|
||||||
|
|
||||||
|
if [ $? -ne 0]; then
|
||||||
|
echo "Unable to retrieve service IPs for {{ $service }}, will retry configuration render."
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
DEST_PORT=$(kubectl \
|
DEST_PORT=$(kubectl \
|
||||||
--server "$KUBE_URL" \
|
--server "$KUBE_URL" \
|
||||||
--certificate-authority "$KUBE_CA" \
|
--certificate-authority "$KUBE_CA" \
|
||||||
|
@ -59,11 +66,18 @@ install_config() {
|
||||||
--namespace {{ $namespace }} \
|
--namespace {{ $namespace }} \
|
||||||
get endpoints {{ $service }} \
|
get endpoints {{ $service }} \
|
||||||
-o 'jsonpath={.subsets[0].ports[0].port}')
|
-o 'jsonpath={.subsets[0].ports[0].port}')
|
||||||
|
|
||||||
|
if [ $? -ne 0]; then
|
||||||
|
echo "Unable to retrieve service port for {{ $service }}, will retry configuration render."
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
set -x
|
set -x
|
||||||
|
|
||||||
if [ "x$SERVICE_IPS" != "x" ]; then
|
if [ "x$SERVICE_IPS" != "x" ]; then
|
||||||
if [ "x$DEST_PORT" != "x" ]; then
|
if [ "x$DEST_PORT" != "x" ]; then
|
||||||
IDENTIFIER=$(echo "{{ $namespace }}-{{ $service }}")
|
IDENTIFIER=$(echo "{{ $namespace }}-{{ $service }}")
|
||||||
|
echo "Adding $IDENTIFIER to haproxy config"
|
||||||
# Add frontend config
|
# Add frontend config
|
||||||
echo >> "$NEXT_HAPROXY_CONF"
|
echo >> "$NEXT_HAPROXY_CONF"
|
||||||
echo "frontend ${IDENTIFIER}-fe" >> "$NEXT_HAPROXY_CONF"
|
echo "frontend ${IDENTIFIER}-fe" >> "$NEXT_HAPROXY_CONF"
|
||||||
|
@ -86,6 +100,7 @@ install_config() {
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
|
||||||
for IP in $SERVICE_IPS; do
|
for IP in $SERVICE_IPS; do
|
||||||
|
echo "Adding backend $IP:$DEST_PORT"
|
||||||
echo " server s$IP $IP:$DEST_PORT" {{ $svc_data.server_opts | quote }} >> "$NEXT_HAPROXY_CONF"
|
echo " server s$IP $IP:$DEST_PORT" {{ $svc_data.server_opts | quote }} >> "$NEXT_HAPROXY_CONF"
|
||||||
done
|
done
|
||||||
else
|
else
|
||||||
|
@ -98,6 +113,7 @@ install_config() {
|
||||||
fi
|
fi
|
||||||
{{- end }}
|
{{- end }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
if [ $SUCCESS = 1 ]; then
|
if [ $SUCCESS = 1 ]; then
|
||||||
mkdir -p $(dirname "$HAPROXY_CONF")
|
mkdir -p $(dirname "$HAPROXY_CONF")
|
||||||
|
@ -115,7 +131,10 @@ install_config() {
|
||||||
fi
|
fi
|
||||||
chmod -R go-rwx $(dirname "$HAPROXY_CONF")
|
chmod -R go-rwx $(dirname "$HAPROXY_CONF")
|
||||||
chown -R $RUNASUSER:$RUNASUSER $(dirname "$HAPROXY_CONF")
|
chown -R $RUNASUSER:$RUNASUSER $(dirname "$HAPROXY_CONF")
|
||||||
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
validate_config() {
|
validate_config() {
|
||||||
|
@ -155,9 +174,9 @@ while true; do
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
|
|
||||||
install_config
|
if install_config; then
|
||||||
|
compare_copy_files
|
||||||
compare_copy_files
|
fi
|
||||||
|
|
||||||
sleep {{ .Values.conf.anchor.period }}
|
sleep {{ .Values.conf.anchor.period }}
|
||||||
done
|
done
|
||||||
|
|
|
@ -38,21 +38,40 @@ spec:
|
||||||
value: {{ .Values.conf.haproxy.container_config_dir }}/haproxy.cfg
|
value: {{ .Values.conf.haproxy.container_config_dir }}/haproxy.cfg
|
||||||
- name: LIVE_HAPROXY_CONF
|
- name: LIVE_HAPROXY_CONF
|
||||||
value: /tmp/live_haproxy.cfg
|
value: /tmp/live_haproxy.cfg
|
||||||
|
- name: STAGE_HAPROXY_CONF
|
||||||
|
value: /tmp/stage_haproxy.cfg
|
||||||
command:
|
command:
|
||||||
- /bin/sh
|
- /bin/sh
|
||||||
- -c
|
- -c
|
||||||
- |
|
- |
|
||||||
set -eux
|
set -eux
|
||||||
|
|
||||||
while [ ! -s "$HAPROXY_CONF" ]; do
|
test_conf () {
|
||||||
echo Waiting for "HAPROXY_CONF"
|
if [ ! -s "$HAPROXY_CONF" ]; then
|
||||||
|
echo "New proposed config not found at $HAPROXY_CONF"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
# this is a safety gate to avoid a race of the anchor
|
||||||
|
# changing a possible new config between the validation
|
||||||
|
# and installation
|
||||||
|
echo "Staging proposed config for installation."
|
||||||
|
cp "$HAPROXY_CONF" "$STAGE_HAPROXY_CONF"
|
||||||
|
if [ ! haproxy -c -f "$STAGE_HAPROXY_CONF"]; then
|
||||||
|
echo "Proposed config not valid."
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
while ! test_conf; do
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
|
|
||||||
echo vvv Starting with initial config vvv
|
echo vvv Starting with initial config vvv
|
||||||
cat "$HAPROXY_CONF"
|
cat "$STAGE_HAPROXY_CONF"
|
||||||
echo
|
echo
|
||||||
cp "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"
|
mv "$STAGE_HAPROXY_CONF" "$LIVE_HAPROXY_CONF"
|
||||||
chmod 700 $LIVE_HAPROXY_CONF
|
chmod 700 "$LIVE_HAPROXY_CONF"
|
||||||
|
|
||||||
# NOTE(mark-burnett): sleep for clearer log output
|
# NOTE(mark-burnett): sleep for clearer log output
|
||||||
sleep 1
|
sleep 1
|
||||||
|
@ -64,7 +83,7 @@ spec:
|
||||||
set +x
|
set +x
|
||||||
while true; do
|
while true; do
|
||||||
if ! cmp -s "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"; then
|
if ! cmp -s "$HAPROXY_CONF" "$LIVE_HAPROXY_CONF"; then
|
||||||
if ! haproxy -c -f "$HAPROXY_CONF"; then
|
if ! test_conf; then
|
||||||
echo New config file appears invalid, refusing to replace.
|
echo New config file appears invalid, refusing to replace.
|
||||||
else
|
else
|
||||||
echo vvv Replacing old config vvv
|
echo vvv Replacing old config vvv
|
||||||
|
@ -72,10 +91,10 @@ spec:
|
||||||
echo
|
echo
|
||||||
|
|
||||||
echo vvv With new config vvv
|
echo vvv With new config vvv
|
||||||
cat "$HAPROXY_CONF"
|
cat "$STAGE_HAPROXY_CONF"
|
||||||
echo
|
echo
|
||||||
|
|
||||||
cat "$HAPROXY_CONF" > "$LIVE_HAPROXY_CONF"
|
cp "$STAGE_HAPROXY_CONF" "$LIVE_HAPROXY_CONF"
|
||||||
|
|
||||||
# NOTE(mark-burnett): sleep for clearer log output
|
# NOTE(mark-burnett): sleep for clearer log output
|
||||||
sleep 1
|
sleep 1
|
||||||
|
|
Loading…
Reference in New Issue