diff --git a/charts/etcd/templates/bin/_etcdctl_anchor.tpl b/charts/etcd/templates/bin/_etcdctl_anchor.tpl index 0562b770..93c1c1e9 100644 --- a/charts/etcd/templates/bin/_etcdctl_anchor.tpl +++ b/charts/etcd/templates/bin/_etcdctl_anchor.tpl @@ -61,6 +61,7 @@ cleanup_host () { firstrun=true saddness_duration=0 while true; do + date # TODO(mark-burnett) Need to monitor a file(s) when shutting down/starting # up so I don't try to take two actions on the node at once. {{- if .Values.bootstrapping.enabled }} @@ -117,7 +118,10 @@ while true; do fi echo Successfully added $HOSTNAME to cluster members. # Refresh member list so we start with the right configuration. - etcdctl member list > /tmp/members + if ! etcdctl member list > /tmp/members; then + echo Could not get a member list, trying again. + continue + fi elif grep $PEER_ENDPOINT /tmp/members | grep '\bunstarted\b'; then # This member is in the cluster but not started if [ $saddness_duration -ge {{ .Values.anchor.saddness_threshold }} ] diff --git a/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl b/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl index 5df4eb10..2ee84805 100644 --- a/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl +++ b/charts/etcd/templates/etc/_kubernetes-etcd.yaml.tpl @@ -16,22 +16,14 @@ {{- $envAll := . }} {{- define "etcdreadinessProbeTemplate" }} -exec: - command: - - /bin/sh - - -c - - |- - etcdctl endpoint health - exit $? +# There's no designated service for readiness check via grpc +# This probe will be unused +grpc: + port: {{ .Values.network.service_client.port }} {{- end }} {{- define "etcdlivenessProbeTemplate" }} -exec: - command: - - /bin/sh - - -c - - |- - etcdctl endpoint status - exit $? +grpc: + port: {{ .Values.network.service_client.port }} {{- end }} # Strip off "etcd" from service name to get the application name # Note that application can either be kubernetes or calico for now @@ -123,36 +115,13 @@ spec: - name: MANIFEST_PATH value: /manifests/{{ .Values.service.name }}.yaml {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.etcd | indent 8 }} +# {{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "readiness" "probeTemplate" (include "etcdreadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }} +{{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "liveness" "probeTemplate" (include "etcdlivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }} volumeMounts: - name: data mountPath: /var/lib/etcd - name: etc mountPath: /etc/etcd - - name: etcd-health-check - image: {{ .Values.images.tags.etcdctl }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.etcd_pod_sidecar | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} -{{ dict "envAll" $envAll "application" "etcd" "container" "etcd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} - env: - - name: ETCDCTL_API - value: "{{ .Values.etcd.etcdctl_api }}" - - name: ETCDCTL_DIAL_TIMEOUT - value: "3s" - - name: ETCDCTL_ENDPOINTS - value: "https://127.0.0.1:{{ .Values.network.service_client.target_port }}" - - name: ETCDCTL_CACERT - value: "/etc/etcd/tls/client-ca.pem" - - name: ETCDCTL_CERT - value: "/etc/etcd/tls/etcd-client.pem" - - name: ETCDCTL_KEY - value: "/etc/etcd/tls/etcd-client-key.pem" - command: ["/bin/sh", "-c", "--"] - args: ["while true; do sleep 30; done;"] -{{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "readiness" "probeTemplate" (include "etcdreadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }} -{{ dict "envAll" $envAll "component" "etcd" "container" "etcd" "type" "liveness" "probeTemplate" (include "etcdlivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 6 }} - volumeMounts: - - name: etc - mountPath: /etc/etcd volumes: - name: data hostPath: diff --git a/charts/etcd/values.yaml b/charts/etcd/values.yaml index 739db2c8..389b1b93 100644 --- a/charts/etcd/values.yaml +++ b/charts/etcd/values.yaml @@ -261,13 +261,6 @@ pod: limits: memory: "1024Mi" cpu: "2000m" - etcd_pod_sidecar: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "1024Mi" - cpu: "2000m" test: limits: memory: "128Mi" diff --git a/promenade/templates/include/genesis-etcd/server-container.yaml b/promenade/templates/include/genesis-etcd/server-container.yaml index a8d64078..de703609 100644 --- a/promenade/templates/include/genesis-etcd/server-container.yaml +++ b/promenade/templates/include/genesis-etcd/server-container.yaml @@ -55,29 +55,37 @@ value: $(ETCD_KEY_FILE) {%- if config['Genesis:etcd.heartbeat_interval'] is defined %} - name: ETCD_HEARTBEAT_INTERVAL - value: {{ config['Genesis:etcd.heartbeat_interval'] }} + value: "{{ config['Genesis:etcd.heartbeat_interval'] }}" {%- endif %} {%- if config['Genesis:etcd.election_timeout'] is defined %} - name: ETCD_ELECTION_TIMEOUT - value: {{ config['Genesis:etcd.election_timeout'] }} + value: "{{ config['Genesis:etcd.election_timeout'] }}" {%- endif %} {%- if config['Genesis:etcd.snapshot_count'] is defined %} - name: ETCD_SNAPSHOT_COUNT - value: {{ config['Genesis:etcd.snapshot_count'] }} + value: "{{ config['Genesis:etcd.snapshot_count'] }}" {%- endif %} {%- if config['Genesis:etcd.cipher_suites'] is defined %} - name: ETCD_CIPHER_SUITES - value: {{ config['Genesis:etcd.cipher_suites'] }} + value: "{{ config['Genesis:etcd.cipher_suites'] }}" {%- endif %} {%- if config['Genesis:etcd.gomaxprocs'] is defined %} - name: GOMAXPROCS - value: {{ config['Genesis:etcd.gomaxprocs'] }} + value: "{{ config['Genesis:etcd.gomaxprocs'] }}" {%- endif %} ports: - name: client containerPort: {{ client_port }} - name: peer containerPort: {{ peer_port }} + livenessProbe: + grpc: + port: {{ client_port }} + initialDelaySeconds: 15 + timeoutSeconds: 10 + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 volumeMounts: - name: data-{{ etcd_name }} mountPath: /var/lib/etcd diff --git a/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml b/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml index 6ee6e770..1facb81d 100644 --- a/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml +++ b/promenade/templates/roles/genesis/etc/kubernetes/manifests/auxiliary-kubernetes-etcd.yaml @@ -38,14 +38,62 @@ spec: MEMBER_NAME=$1 MEMBER_ID=$(etcdctl member list | grep "${MEMBER_NAME}" | awk -F ', ' '{ print $1 }') if [ -n "${MEMBER_ID}" ]; then + if is_leader "$MEMBER_ID"; then + abdicate + fi etcdctl member remove $MEMBER_ID + sleep 10 fi } + adjust_endpoints () { + ENDPOINTS=$(echo $1 | sed 's/\//\\\//g') + echo "Adjusting ETCD_ENDPOINTS to $1 in apiserver" + # sed -i "/- name: ETCD_ENDPOINTS/{n;s/value: .*/value: \"$ENDPOINTS\"/}" /manifests/kubernetes-apiserver.yaml + } + + abdicate () { + NEW_LEADER=$(etcdctl member list | grep '\bstarted\b' | grep -Ev '\\b(auxiliary-0|auxiliary-1)\\b' | head -1 | awk -F ', ' '{print $1}') + if [ -n "$NEW_LEADER" ]; then + if ! etcdctl move-leader "$NEW_LEADER"; then + echo "Attempted abdication, but failed." + return + fi + sleep 5 + return + fi + } + + abdicate_if_needed () { + AUX_MEMBERS=$(etcdctl member list | grep '\bstarted\b' | grep -E "\\b(auxiliary-0|auxiliary-1)\\b" | awk -F ', ' '{print $1}') + for m in $AUX_MEMBERS; do + if is_leader "$m"; then + abdicate + fi + done + } + + is_leader () { + MEMBER_ID="$1" + MEMBER_EP=$(etcdctl member list | grep "$MEMBER_ID" | awk -F ', ' '{print $5}') + IS_LEADER=$(ETCDCTL_ENDPOINTS="$MEMBER_EP" etcdctl endpoint status | awk -F ', ' '{ print $5 }') + if [ "$IS_LEADER" = "true" ]; then + return 0 + else + return 1 + fi + } + auxiliary_threshold="{{ config.get_first('Genesis:etcd.auxiliary_threshold', default=3) }}" # NOTE(sh8121att): If there are enough (a fully resilient contigent) non-auxiliary members, # then we are ready to remove the auxiliary members. Otherwise, wait. while [ ! "$(external_member_count)" -ge "$auxiliary_threshold" ]; do + # abdicate_if_needed + CURRENT_ENDPOINTS=$(etcdctl member list | awk -F ',' '{ if (output != "") output=output","$5; else output=$5} END {gsub(/ /, "", output); print output}') + APISERVER_ENDPOINTS=$(awk '/- name: ETCD_ENDPOINTS/{getline; sub(/.*: "/, ""); sub(/".*/, ""); print}' /manifests/kubernetes-apiserver.yaml) + if [ "$CURRENT_ENDPOINTS" != "$APISERVER_ENDPOINTS" ]; then + adjust_endpoints $CURRENT_ENDPOINTS + fi sleep 30 done @@ -53,6 +101,11 @@ spec: # should be recovered by restarting this container. set -e + sleep 600 + # make a switch to permanent etcd cluster + PERMANENT_ENDPOINTS=$(etcdctl member list | awk -F ',' '$3 ~! /auxiliary/ { if (output != "") output=output","$5; else output=$5} END {gsub(/ /, "", output); print output}' | sed 's/\//\\\//g') + adjust_endpoints $PERMANENT_ENDPOINTS + remove_if_possible auxiliary-0 remove_if_possible auxiliary-1