diff --git a/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl b/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl index 5df4eb10..6dd80e08 100644 --- a/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl +++ b/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl @@ -16,22 +16,16 @@ {{- $envAll := . }} {{- define "etcdreadinessProbeTemplate" }} -exec: - command: - - /bin/sh - - -c - - |- - etcdctl endpoint health - exit $? +httpGet: + path: /readyz + port: {{ .Values.network.service_client.port }} + scheme: HTTPS {{- end }} {{- define "etcdlivenessProbeTemplate" }} -exec: - command: - - /bin/sh - - -c - - |- - etcdctl endpoint status - exit $? +httpGet: + path: /livez + port: {{ .Values.network.service_client.port }} + scheme: HTTPS {{- end }} # Strip off "etcd" from service name to get the application name # Note that application can either be kubernetes or calico for now @@ -123,34 +117,11 @@ spec: - name: MANIFEST_PATH value: /manifests/{{ .Values.service.name }}.yaml {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.etcd | indent 8 }} - volumeMounts: - - name: data - mountPath: /var/lib/etcd - - name: etc - mountPath: /etc/etcd - - name: etcd-health-check - image: {{ .Values.images.tags.etcdctl }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.etcd_pod_sidecar | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} -{{ dict "envAll" $envAll "application" "etcd" "container" "etcd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} - env: - - name: ETCDCTL_API - value: "{{ .Values.etcd.etcdctl_api }}" - - name: ETCDCTL_DIAL_TIMEOUT - value: "3s" - - name: ETCDCTL_ENDPOINTS - value: "https://127.0.0.1:{{ .Values.network.service_client.target_port }}" - - name: ETCDCTL_CACERT - value: "/etc/etcd/tls/client-ca.pem" - - name: ETCDCTL_CERT - value: "/etc/etcd/tls/etcd-client.pem" - - name: ETCDCTL_KEY - value: "/etc/etcd/tls/etcd-client-key.pem" - command: ["/bin/sh", "-c", "--"] - args: ["while true; do sleep 30; done;"] {{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "readiness" "probeTemplate" (include "etcdreadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }} {{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "liveness" "probeTemplate" (include "etcdlivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }} volumeMounts: + - name: data + mountPath: /var/lib/etcd - name: etc mountPath: /etc/etcd volumes: diff --git a/charts/etcd/values.yaml b/charts/etcd/values.yaml index 739db2c8..389b1b93 100644 --- a/charts/etcd/values.yaml +++ b/charts/etcd/values.yaml @@ -261,13 +261,6 @@ pod: limits: memory: "1024Mi" cpu: "2000m" - etcd_pod_sidecar: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "1024Mi" - cpu: "2000m" test: limits: memory: "128Mi" diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml index 6ee6e770..e3e25cde 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml @@ -39,13 +39,25 @@ spec: MEMBER_ID=$(etcdctl member list | grep "${MEMBER_NAME}" | awk -F ', ' '{ print $1 }') if [ -n "${MEMBER_ID}" ]; then etcdctl member remove $MEMBER_ID + sleep 10 fi } + + adjust_endpoints () { + ENDPOINTS=$1 + echo "Adjusting ETCD_ENDPOINTS to $ENDPOINTS in apiserver" + sed -i "/- name: ETCD_ENDPOINTS/{n;s/value: .*/value: \"$ENDPOINTS\"/}" /manifests/kubernetes-apiserver.yaml + } auxiliary_threshold="{{ config.get_first('Genesis:etcd.auxiliary_threshold', default=3) }}" # NOTE(sh8121att): If there are enough (a fully resilient contigent) non-auxiliary members, # then we are ready to remove the auxiliary members. Otherwise, wait. while [ ! "$(external_member_count)" -ge "$auxiliary_threshold" ]; do + CURRENT_ENDPOINTS=$(etcdctl member list | awk -F ',' '{ if (output != "") output=output","$5; else output=$5} END {gsub(/ /, "", output); print output}') + APISERVER_ENDPOINTS=$(awk '/- name: ETCD_ENDPOINTS/{getline; sub(/.*: "/, ""); sub(/".*/, ""); print}' /manifests/kubernetes-apiserver.yaml) + if [ "$CURRENT_ENDPOINTS" != "$APISERVER_ENDPOINTS" ]; then + adjust_endpoints $CURRENT_ENDPOINTS + fi sleep 30 done @@ -53,6 +65,11 @@ spec: # should be recovered by restarting this container. set -e + sleep 600 + # make a switch to permanent etcd cluster + PERMANENT_ENDPOINTS=$(etcdctl member list | awk -F ',' '$3 ~! /auxiliary/ { if (output != "") output=output","$5; else output=$5} END {gsub(/ /, "", output); print output}' | sed 's/\//\\\//g') + adjust_endpoints $PERMANENT_ENDPOINTS + remove_if_possible auxiliary-0 remove_if_possible auxiliary-1