diff --git a/charts/etcd/templates/bin/_etcdctl_anchor.tpl b/charts/etcd/templates/bin/_etcdctl_anchor.tpl index 0562b770..93c1c1e9 100644 --- a/charts/etcd/templates/bin/_etcdctl_anchor.tpl +++ b/charts/etcd/templates/bin/_etcdctl_anchor.tpl @@ -61,6 +61,7 @@ cleanup_host () { firstrun=true saddness_duration=0 while true; do + date # TODO(mark-burnett) Need to monitor a file(s) when shutting down/starting # up so I don't try to take two actions on the node at once. {{- if .Values.bootstrapping.enabled }} @@ -117,7 +118,10 @@ while true; do fi echo Successfully added $HOSTNAME to cluster members. # Refresh member list so we start with the right configuration. - etcdctl member list > /tmp/members + if ! etcdctl member list > /tmp/members; then + echo Could not get a member list, trying again. + continue + fi elif grep $PEER_ENDPOINT /tmp/members | grep '\bunstarted\b'; then # This member is in the cluster but not started if [ $saddness_duration -ge {{ .Values.anchor.saddness_threshold }} ] diff --git a/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl b/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl index 5df4eb10..d942aa90 100644 --- a/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl +++ b/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl @@ -16,22 +16,12 @@ {{- $envAll := . }} {{- define "etcdreadinessProbeTemplate" }} -exec: - command: - - /bin/sh - - -c - - |- - etcdctl endpoint health - exit $? +grpc: + port: {{ .Values.network.service_client.port }} {{- end }} {{- define "etcdlivenessProbeTemplate" }} -exec: - command: - - /bin/sh - - -c - - |- - etcdctl endpoint status - exit $? +grpc: + port: {{ .Values.network.service_client.port }} {{- end }} # Strip off "etcd" from service name to get the application name # Note that application can either be kubernetes or calico for now @@ -123,34 +113,11 @@ spec: - name: MANIFEST_PATH value: /manifests/{{ .Values.service.name }}.yaml {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.etcd | indent 8 }} - volumeMounts: - - name: data - mountPath: /var/lib/etcd - - name: etc - mountPath: /etc/etcd - - name: etcd-health-check - image: {{ .Values.images.tags.etcdctl }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.etcd_pod_sidecar | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} -{{ dict "envAll" $envAll "application" "etcd" "container" "etcd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} - env: - - name: ETCDCTL_API - value: "{{ .Values.etcd.etcdctl_api }}" - - name: ETCDCTL_DIAL_TIMEOUT - value: "3s" - - name: ETCDCTL_ENDPOINTS - value: "https://127.0.0.1:{{ .Values.network.service_client.target_port }}" - - name: ETCDCTL_CACERT - value: "/etc/etcd/tls/client-ca.pem" - - name: ETCDCTL_CERT - value: "/etc/etcd/tls/etcd-client.pem" - - name: ETCDCTL_KEY - value: "/etc/etcd/tls/etcd-client-key.pem" - command: ["/bin/sh", "-c", "--"] - args: ["while true; do sleep 30; done;"] {{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "readiness" "probeTemplate" (include "etcdreadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }} {{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "liveness" "probeTemplate" (include "etcdlivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }} volumeMounts: + - name: data + mountPath: /var/lib/etcd - name: etc mountPath: /etc/etcd volumes: diff --git a/charts/etcd/values.yaml b/charts/etcd/values.yaml index 739db2c8..389b1b93 100644 --- a/charts/etcd/values.yaml +++ b/charts/etcd/values.yaml @@ -261,13 +261,6 @@ pod: limits: memory: "1024Mi" cpu: "2000m" - etcd_pod_sidecar: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "1024Mi" - cpu: "2000m" test: limits: memory: "128Mi" diff --git a/promenade/templates/include/genesis-etcd/server-container.yaml b/promenade/templates/include/genesis-etcd/server-container.yaml index a8d64078..9579a5ff 100644 --- a/promenade/templates/include/genesis-etcd/server-container.yaml +++ b/promenade/templates/include/genesis-etcd/server-container.yaml @@ -55,29 +55,45 @@ value: $(ETCD_KEY_FILE) {%- if config['Genesis:etcd.heartbeat_interval'] is defined %} - name: ETCD_HEARTBEAT_INTERVAL - value: {{ config['Genesis:etcd.heartbeat_interval'] }} + value: "{{ config['Genesis:etcd.heartbeat_interval'] }}" {%- endif %} {%- if config['Genesis:etcd.election_timeout'] is defined %} - name: ETCD_ELECTION_TIMEOUT - value: {{ config['Genesis:etcd.election_timeout'] }} + value: "{{ config['Genesis:etcd.election_timeout'] }}" {%- endif %} {%- if config['Genesis:etcd.snapshot_count'] is defined %} - name: ETCD_SNAPSHOT_COUNT - value: {{ config['Genesis:etcd.snapshot_count'] }} + value: "{{ config['Genesis:etcd.snapshot_count'] }}" {%- endif %} {%- if config['Genesis:etcd.cipher_suites'] is defined %} - name: ETCD_CIPHER_SUITES - value: {{ config['Genesis:etcd.cipher_suites'] }} + value: "{{ config['Genesis:etcd.cipher_suites'] }}" {%- endif %} {%- if config['Genesis:etcd.gomaxprocs'] is defined %} - name: GOMAXPROCS - value: {{ config['Genesis:etcd.gomaxprocs'] }} + value: "{{ config['Genesis:etcd.gomaxprocs'] }}" {%- endif %} ports: - name: client containerPort: {{ client_port }} - name: peer containerPort: {{ peer_port }} + livenessProbe: + grpc: + port: {{ client_port }} + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + grpc: + port: {{ client_port }} + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 volumeMounts: - name: data-{{ etcd_name }} mountPath: /var/lib/etcd diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml index 6ee6e770..1dcb6967 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml @@ -38,14 +38,56 @@ spec: MEMBER_NAME=$1 MEMBER_ID=$(etcdctl member list | grep "${MEMBER_NAME}" | awk -F ', ' '{ print $1 }') if [ -n "${MEMBER_ID}" ]; then + if is_leader "$MEMBER_ID"; then + abdicate + fi etcdctl member remove $MEMBER_ID + sleep 10 fi } + abdicate () { + NEW_LEADER=$(etcdctl member list | grep '\bstarted\b' | grep -Ev '\\b(auxiliary-0|auxiliary-1)\\b' | head -1 | awk -F ', ' '{print $1}') + if [ -n "$NEW_LEADER" ]; then + if ! etcdctl move-leader "$NEW_LEADER"; then + echo "Attempted abdication, but failed." + return + fi + sleep 5 + return + fi + } + + abdicate_if_needed () { + AUX_MEMBERS=$(etcdctl member list | grep '\bstarted\b' | grep -E "\\b(auxiliary-0|auxiliary-1)\\b" | awk -F ', ' '{print $1}') + for m in $AUX_MEMBERS; do + if is_leader "$m"; then + abdicate + fi + done + } + + is_leader () { + MEMBER_ID="$1" + MEMBER_EP=$(etcdctl member list | grep "$MEMBER_ID" | awk -F ', ' '{print $5}') + IS_LEADER=$(ETCDCTL_ENDPOINTS="$MEMBER_EP" etcdctl endpoint status | awk -F ', ' '{ print $5 }') + if [ "$IS_LEADER" = "true" ]; then + return 0 + else + return 1 + fi + } + + aux_endpoint_present () { + awk '/- name: ETCD_ENDPOINTS/{getline; sub(/.*: "/, ""); sub(/".*/, ""); print}' /manifests/kubernetes-apiserver.yaml \ + | grep -E "\\b(12379|22379)\\b" + } + auxiliary_threshold="{{ config.get_first('Genesis:etcd.auxiliary_threshold', default=3) }}" # NOTE(sh8121att): If there are enough (a fully resilient contigent) non-auxiliary members, # then we are ready to remove the auxiliary members. Otherwise, wait. - while [ ! "$(external_member_count)" -ge "$auxiliary_threshold" ]; do + while [ ! "$(external_member_count)" -ge "$auxiliary_threshold" ] || [ "$(aux_endpoint_present)" ]; do + abdicate_if_needed sleep 30 done