From 2885218d35f3bc73722638f14a500b69b7197e1c Mon Sep 17 00:00:00 2001 From: Aaron Sheffield Date: Wed, 11 Apr 2018 13:33:58 -0500 Subject: [PATCH] Add Helm test for HAProxy - Added a helm test to test HAProxy's health via kubernetes healthz endpoint. Change-Id: I0ffba39d4e4245fad69c27f0fcafdcb58fdc9067 --- .../etc/_kubernetes-apiserver.yaml.tpl | 2 +- .../templates/tests/test-haproxy-health.yaml | 42 +++++++++++++++++++ charts/haproxy/values.yaml | 20 +++++++-- .../templates/tests/test-promenade-api.yaml | 3 +- examples/basic/armada-resources.yaml | 7 ++-- examples/complete/armada-resources.yaml | 7 ++-- promenade/templates/include/utils.sh | 2 +- .../common/etc/promenade/haproxy/haproxy.cfg | 1 - .../manifests/bootstrap-armada.yaml | 2 +- .../manifests/kubernetes-apiserver.yaml | 2 +- .../bootstrap-armada-config.yaml | 4 -- 11 files changed, 70 insertions(+), 22 deletions(-) create mode 100644 charts/haproxy/templates/tests/test-haproxy-health.yaml diff --git a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl index 125185ab..d09b3799 100644 --- a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl +++ b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl @@ -38,7 +38,7 @@ spec: - {{ . }} {{- end }} - --advertise-address=$(POD_IP) - - --anonymous-auth=false + - --anonymous-auth=true - --bind-address=0.0.0.0 - --secure-port={{ .Values.network.kubernetes_apiserver.port }} - --insecure-port=0 diff --git a/charts/haproxy/templates/tests/test-haproxy-health.yaml b/charts/haproxy/templates/tests/test-haproxy-health.yaml new file mode 100644 index 00000000..bc81891f --- /dev/null +++ b/charts/haproxy/templates/tests/test-haproxy-health.yaml @@ -0,0 +1,42 @@ +{{/* +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{/* +Test that HAProxy is running and the config is valid */}} +{{- if .Values.manifests.test_haproxy }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{ .Release.Name }}-haproxy-test" + annotations: + "helm.sh/hook": "test-success" +spec: + restartPolicy: Never + containers: + - name: "{{ .Release.Name }}-haproxy-test" + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: 'HAPROXY_URL' + value: https://$(HOST_IP):{{ .Values.endpoints.health.port }}/{{ .Values.endpoints.health.path }} + image: {{ .Values.images.tags.test }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple . .Values.pod.resources.test | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} + command: ["/bin/sh", "-c", "wget --no-check-certificate --spider ${HAPROXY_URL}; exit $?"] +... +{{- end }} diff --git a/charts/haproxy/values.yaml b/charts/haproxy/values.yaml index 2b325a86..b8d16a6e 100644 --- a/charts/haproxy/values.yaml +++ b/charts/haproxy/values.yaml @@ -26,11 +26,9 @@ conf: conf_parts: frontend: - mode tcp - - option tcpka - bind *:6553 backend: - mode tcp - - option tcpka - option tcp-check - option redispatch kube-system: @@ -39,11 +37,9 @@ conf: conf_parts: frontend: - mode tcp - - option tcpka - bind *:2378 backend: - mode tcp - - option tcpka - option tcp-check - option redispatch @@ -65,8 +61,17 @@ images: tags: anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6 haproxy: haproxy:1.8.3 + test: busybox:1.28.3 pull_policy: "IfNotPresent" +manifests: + test_haproxy: true + +endpoints: + health: + port: 6553 + path: "healthz" + pod: lifecycle: upgrades: @@ -95,5 +100,12 @@ pod: limits: memory: "256Mi" cpu: "2000m" + test: + limits: + memory: "128Mi" + cpu: "100m" + requests: + memory: "128Mi" + cpu: "100m" release_group: null diff --git a/charts/promenade/templates/tests/test-promenade-api.yaml b/charts/promenade/templates/tests/test-promenade-api.yaml index 6791f66e..80659eb3 100644 --- a/charts/promenade/templates/tests/test-promenade-api.yaml +++ b/charts/promenade/templates/tests/test-promenade-api.yaml @@ -11,7 +11,8 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. */}} +# limitations under the License. +*/}} {{/* Test that the API is up and the health endpoint returns a 2XX code */}} {{- if .Values.manifests.test_promenade_api }} diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index 7c2c3144..8771f123 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -572,6 +572,7 @@ data: timeout: 600 wait: timeout: 600 + test: true upgrade: no_hooks: true values: @@ -585,11 +586,9 @@ data: conf_parts: frontend: - mode tcp - - option tcpka - bind *:6553 backend: - mode tcp - - option tcpka - option tcp-check - option redispatch kube-system: @@ -598,11 +597,9 @@ data: conf_parts: frontend: - mode tcp - - option tcpka - bind *:2378 backend: - mode tcp - - option tcpka - option tcp-check - option redispatch @@ -610,6 +607,7 @@ data: tags: anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6 haproxy: haproxy:1.8.3 + test: busybox:1.28.3 source: type: local @@ -1139,6 +1137,7 @@ data: timeout: 600 wait: timeout: 600 + test: true values: pod: env: diff --git a/examples/complete/armada-resources.yaml b/examples/complete/armada-resources.yaml index f25228b0..88ee01c2 100644 --- a/examples/complete/armada-resources.yaml +++ b/examples/complete/armada-resources.yaml @@ -612,6 +612,7 @@ data: timeout: 600 wait: timeout: 600 + test: true upgrade: no_hooks: true values: @@ -625,11 +626,9 @@ data: conf_parts: frontend: - mode tcp - - option tcpka - bind *:6553 backend: - mode tcp - - option tcpka - option tcp-check - option redispatch kube-system: @@ -638,11 +637,9 @@ data: conf_parts: frontend: - mode tcp - - option tcpka - bind *:2378 backend: - mode tcp - - option tcpka - option tcp-check - option redispatch @@ -650,6 +647,7 @@ data: tags: anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.6 haproxy: haproxy:1.8.3 + test: busybox:1.28.3 source: type: local @@ -1816,6 +1814,7 @@ data: timeout: 600 wait: timeout: 600 + test: true values: pod: env: diff --git a/promenade/templates/include/utils.sh b/promenade/templates/include/utils.sh index cb79e81f..7c846236 100644 --- a/promenade/templates/include/utils.sh +++ b/promenade/templates/include/utils.sh @@ -234,7 +234,7 @@ spec: kubernetes.io/hostname: ${NODE} containers: - name: noisy - image: busybox:1.27.1 + image: busybox:1.28.3 imagePullPolicy: IfNotPresent command: - /bin/echo diff --git a/promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg b/promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg index bf304dae..d6518775 100644 --- a/promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg +++ b/promenade/templates/roles/common/etc/promenade/haproxy/haproxy.cfg @@ -17,7 +17,6 @@ global defaults mode tcp - option tcpka timeout connect 5000ms timeout client 24h timeout server 24h diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml index 40d98329..0b55efa3 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml @@ -123,7 +123,7 @@ spec: - --advertise-address={{ config['Genesis:ip'] }} - --authorization-mode=Node,RBAC - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds - - --anonymous-auth=false + - --anonymous-auth=true - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml index 37218b18..e9b7bb8a 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/kubernetes-apiserver.yaml @@ -19,7 +19,7 @@ spec: - --advertise-address={{ config['Genesis:ip'] }} - --authorization-mode=Node,RBAC - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds - - --anonymous-auth=false + - --anonymous-auth=true - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/cluster-ca.pem - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/apiserver.pem diff --git a/tools/gate/config-templates/bootstrap-armada-config.yaml b/tools/gate/config-templates/bootstrap-armada-config.yaml index c915596e..f4c88f2d 100644 --- a/tools/gate/config-templates/bootstrap-armada-config.yaml +++ b/tools/gate/config-templates/bootstrap-armada-config.yaml @@ -564,11 +564,9 @@ data: conf_parts: frontend: - mode tcp - - option tcpka - bind *:6553 backend: - mode tcp - - option tcpka - option tcp-check - option redispatch kube-system: @@ -577,11 +575,9 @@ data: conf_parts: frontend: - mode tcp - - option tcpka - bind *:2378 backend: - mode tcp - - option tcpka - option tcp-check - option redispatch