summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMark Burnett <mark.m.burnett@gmail.com>2018-10-10 10:02:45 -0500
committerMark Burnett <mark.m.burnett@gmail.com>2018-10-10 10:02:45 -0500
commiteaeb3ae250d9cf29101799a4dd2c041d6d1f51e7 (patch)
tree87e6e92c7f5980a52b65f9a151fe9847d17affdc
parent83b65b358d10cde53446a8bb33048c9c9e40c017 (diff)
Make kube-proxy liveness probe more cautious
This update makes it so list of services without endpoints detected on the host must be static to cause failure. This avoids race conditions for large deployments where new services are being added over several minutes, and trigger probe failures. Change-Id: Ie65c8613cb85bfdf61d41099540d3499ea1de817
Notes
Notes (review): Code-Review+2: Aaron Sheffield <ajs@sheffieldfamily.net> Code-Review+2: Bryan Strassner <bryan.strassner@gmail.com> Workflow+1: Bryan Strassner <bryan.strassner@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Wed, 10 Oct 2018 19:03:10 +0000 Reviewed-on: https://review.openstack.org/609443 Project: openstack/airship-promenade Branch: refs/heads/master
-rw-r--r--charts/proxy/templates/bin/_liveness-probe.sh.tpl22
1 files changed, 17 insertions, 5 deletions
diff --git a/charts/proxy/templates/bin/_liveness-probe.sh.tpl b/charts/proxy/templates/bin/_liveness-probe.sh.tpl
index 81cfa56..e76f4c9 100644
--- a/charts/proxy/templates/bin/_liveness-probe.sh.tpl
+++ b/charts/proxy/templates/bin/_liveness-probe.sh.tpl
@@ -2,6 +2,8 @@
2 2
3set -e 3set -e
4 4
5IPTS_DIR=/tmp/liveness
6
5FAILURE=0 7FAILURE=0
6{{- if .Values.livenessProbe.whitelist }} 8{{- if .Values.livenessProbe.whitelist }}
7WHITELIST='({{- join "|" .Values.livenessProbe.whitelist -}})' 9WHITELIST='({{- join "|" .Values.livenessProbe.whitelist -}})'
@@ -15,12 +17,23 @@ if [[ $(echo -e "${REQUEST}" | socat - TCP4:localhost:10256 | grep -sc '200 OK')
15 FAILURE=1 17 FAILURE=1
16fi 18fi
17 19
18if [[ $(iptables-save {{- if .Values.livenessProbe.whitelist }} | grep -Ev "${WHITELIST}" {{- end }} | grep -sc 'has no endpoints') -gt 0 ]]; then 20mkdir -p "${IPTS_DIR}"
19 echo Some non-whitelisted services have no endpoints: 21iptables-save {{- if .Values.livenessProbe.whitelist }} | grep -Ev "${WHITELIST}" {{- end }} | grep -s 'has no endpoints' | sort > "${IPTS_DIR}/current"
20 iptables-save | grep 'has no endpoints' 22
21 FAILURE=1 23if [[ $(wc -l "${IPTS_DIR}/current") -gt 0 ]]; then
24 if [[ "${IPTS_DIR}/previous" ]]; then
25 if cmp "${IPTS_DIR}/current" "${IPTS_DIR}/previous"; then
26 echo Some non-whitelisted services have no endpoints:
27 cat "${IPTS_DIR}/current"
28 FAILURE=1
29 else
30 echo Detected issues have changed. Passing check:
31 diff "${IPTS_DIR}/previous" "${IPTS_DIR}/current"
32 fi
33 fi
22fi 34fi
23 35
36mv "${IPTS_DIR}/current" "${IPTS_DIR}/previous"
24 37
25IPTABLES_IPS=$(iptables-save | grep -E 'KUBE-SEP.*to-destination' | sed 's/.*to-destination \(.*\):.*/\1/' | sort -u) 38IPTABLES_IPS=$(iptables-save | grep -E 'KUBE-SEP.*to-destination' | sed 's/.*to-destination \(.*\):.*/\1/' | sort -u)
26KUBECTL_IPS=$(kubectl get --all-namespaces -o json endpoints | jq -r '.items | arrays | .[] | objects | .subsets | arrays | .[] | objects | .addresses | arrays | .[] | objects | .ip' | sort -u) 39KUBECTL_IPS=$(kubectl get --all-namespaces -o json endpoints | jq -r '.items | arrays | .[] | objects | .subsets | arrays | .[] | objects | .addresses | arrays | .[] | objects | .ip' | sort -u)
@@ -31,7 +44,6 @@ if [[ $(comm -23 <(echo "${IPTABLES_IPS}") <(echo "${KUBECTL_IPS}")) ]]; then
31 comm -23 <(echo "${IPTABLES_IPS}") <(echo "${KUBECTL_IPS}") 44 comm -23 <(echo "${IPTABLES_IPS}") <(echo "${KUBECTL_IPS}")
32fi 45fi
33 46
34
35if [[ "${FAILURE}" == "1" ]]; then 47if [[ "${FAILURE}" == "1" ]]; then
36 exit 1 48 exit 1
37fi 49fi